1416d8220SZaibo Xu // SPDX-License-Identifier: GPL-2.0
2416d8220SZaibo Xu /* Copyright (c) 2019 HiSilicon Limited. */
3416d8220SZaibo Xu
4416d8220SZaibo Xu #include <crypto/aes.h>
56c46a329SKai Ye #include <crypto/aead.h>
6416d8220SZaibo Xu #include <crypto/algapi.h>
72f072d75SZaibo Xu #include <crypto/authenc.h>
8416d8220SZaibo Xu #include <crypto/des.h>
92f072d75SZaibo Xu #include <crypto/hash.h>
102f072d75SZaibo Xu #include <crypto/internal/aead.h>
11ae6ce7b1SKai Ye #include <crypto/internal/des.h>
12a24d22b2SEric Biggers #include <crypto/sha1.h>
13a24d22b2SEric Biggers #include <crypto/sha2.h>
14416d8220SZaibo Xu #include <crypto/skcipher.h>
15416d8220SZaibo Xu #include <crypto/xts.h>
16416d8220SZaibo Xu #include <linux/crypto.h>
17416d8220SZaibo Xu #include <linux/dma-mapping.h>
18416d8220SZaibo Xu #include <linux/idr.h>
19416d8220SZaibo Xu
20416d8220SZaibo Xu #include "sec.h"
21416d8220SZaibo Xu #include "sec_crypto.h"
22416d8220SZaibo Xu
23416d8220SZaibo Xu #define SEC_PRIORITY 4001
24416d8220SZaibo Xu #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE)
255652d55aSKai Ye #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE)
26416d8220SZaibo Xu #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE)
27416d8220SZaibo Xu #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE)
28416d8220SZaibo Xu #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE)
29416d8220SZaibo Xu
30416d8220SZaibo Xu /* SEC sqe(bd) bit operational relative MACRO */
31416d8220SZaibo Xu #define SEC_DE_OFFSET 1
32416d8220SZaibo Xu #define SEC_CIPHER_OFFSET 4
33416d8220SZaibo Xu #define SEC_SCENE_OFFSET 3
34416d8220SZaibo Xu #define SEC_DST_SGL_OFFSET 2
35416d8220SZaibo Xu #define SEC_SRC_SGL_OFFSET 7
36416d8220SZaibo Xu #define SEC_CKEY_OFFSET 9
37416d8220SZaibo Xu #define SEC_CMODE_OFFSET 12
382f072d75SZaibo Xu #define SEC_AKEY_OFFSET 5
392f072d75SZaibo Xu #define SEC_AEAD_ALG_OFFSET 11
402f072d75SZaibo Xu #define SEC_AUTH_OFFSET 6
412f072d75SZaibo Xu
42adc3f65aSKai Ye #define SEC_DE_OFFSET_V3 9
43adc3f65aSKai Ye #define SEC_SCENE_OFFSET_V3 5
44adc3f65aSKai Ye #define SEC_CKEY_OFFSET_V3 13
45566f060fSKai Ye #define SEC_CTR_CNT_OFFSET 25
46566f060fSKai Ye #define SEC_CTR_CNT_ROLLOVER 2
47adc3f65aSKai Ye #define SEC_SRC_SGL_OFFSET_V3 11
48adc3f65aSKai Ye #define SEC_DST_SGL_OFFSET_V3 14
49adc3f65aSKai Ye #define SEC_CALG_OFFSET_V3 4
50adc3f65aSKai Ye #define SEC_AKEY_OFFSET_V3 9
51adc3f65aSKai Ye #define SEC_MAC_OFFSET_V3 4
52adc3f65aSKai Ye #define SEC_AUTH_ALG_OFFSET_V3 15
53adc3f65aSKai Ye #define SEC_CIPHER_AUTH_V3 0xbf
54adc3f65aSKai Ye #define SEC_AUTH_CIPHER_V3 0x40
55416d8220SZaibo Xu #define SEC_FLAG_OFFSET 7
56416d8220SZaibo Xu #define SEC_FLAG_MASK 0x0780
57416d8220SZaibo Xu #define SEC_TYPE_MASK 0x0F
58416d8220SZaibo Xu #define SEC_DONE_MASK 0x0001
59668f1ab7SKai Ye #define SEC_ICV_MASK 0x000E
60adc3f65aSKai Ye #define SEC_SQE_LEN_RATE_MASK 0x3
61416d8220SZaibo Xu
62129a9f34SWeili Qian #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth))
63416d8220SZaibo Xu #define SEC_SGL_SGE_NR 128
642f072d75SZaibo Xu #define SEC_CIPHER_AUTH 0xfe
652f072d75SZaibo Xu #define SEC_AUTH_CIPHER 0x1
662f072d75SZaibo Xu #define SEC_MAX_MAC_LEN 64
672514f559SLongfang Liu #define SEC_MAX_AAD_LEN 65535
685e340558SKai Ye #define SEC_MAX_CCM_AAD_LEN 65279
69129a9f34SWeili Qian #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))
7074b58db8SLongfang Liu
7174b58db8SLongfang Liu #define SEC_PBUF_SZ 512
7274b58db8SLongfang Liu #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ
7374b58db8SLongfang Liu #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE)
7474b58db8SLongfang Liu #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \
7574b58db8SLongfang Liu SEC_MAX_MAC_LEN * 2)
7674b58db8SLongfang Liu #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG)
77129a9f34SWeili Qian #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM)
78129a9f34SWeili Qian #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \
79129a9f34SWeili Qian SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM))
80129a9f34SWeili Qian #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \
81129a9f34SWeili Qian SEC_PBUF_LEFT_SZ(depth))
8274b58db8SLongfang Liu
832f072d75SZaibo Xu #define SEC_SQE_LEN_RATE 4
84d6de2a59SZaibo Xu #define SEC_SQE_CFLAG 2
852f072d75SZaibo Xu #define SEC_SQE_AEAD_FLAG 3
86d6de2a59SZaibo Xu #define SEC_SQE_DONE 0x1
87668f1ab7SKai Ye #define SEC_ICV_ERR 0x2
88c16a70c1SKai Ye #define MIN_MAC_LEN 4
89c16a70c1SKai Ye #define MAC_LEN_MASK 0x1U
907b44c0eeSKai Ye #define MAX_INPUT_DATA_LEN 0xFFFE00
917b44c0eeSKai Ye #define BITS_MASK 0xFF
927b44c0eeSKai Ye #define BYTE_BITS 0x8
935652d55aSKai Ye #define SEC_XTS_NAME_SZ 0x3
94c16a70c1SKai Ye #define IV_CM_CAL_NUM 2
95c16a70c1SKai Ye #define IV_CL_MASK 0x7
96c16a70c1SKai Ye #define IV_CL_MIN 2
97c16a70c1SKai Ye #define IV_CL_MID 4
98c16a70c1SKai Ye #define IV_CL_MAX 8
99c16a70c1SKai Ye #define IV_FLAGS_OFFSET 0x6
100c16a70c1SKai Ye #define IV_CM_OFFSET 0x3
101c16a70c1SKai Ye #define IV_LAST_BYTE1 1
102c16a70c1SKai Ye #define IV_LAST_BYTE2 2
103c16a70c1SKai Ye #define IV_LAST_BYTE_MASK 0xFF
104c16a70c1SKai Ye #define IV_CTR_INIT 0x1
105c16a70c1SKai Ye #define IV_BYTE_OFFSET 0x8
106416d8220SZaibo Xu
107b42ab1c6SWeili Qian static DEFINE_MUTEX(sec_algs_lock);
108b42ab1c6SWeili Qian static unsigned int sec_available_devs;
109b42ab1c6SWeili Qian
110921715b6SWenkai Lin struct sec_skcipher {
111921715b6SWenkai Lin u64 alg_msk;
112921715b6SWenkai Lin struct skcipher_alg alg;
113921715b6SWenkai Lin };
114921715b6SWenkai Lin
115921715b6SWenkai Lin struct sec_aead {
116921715b6SWenkai Lin u64 alg_msk;
117921715b6SWenkai Lin struct aead_alg alg;
118921715b6SWenkai Lin };
119921715b6SWenkai Lin
120416d8220SZaibo Xu /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
sec_alloc_queue_id(struct sec_ctx * ctx,struct sec_req * req)121dd1a502cSQi Tao static inline u32 sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
122416d8220SZaibo Xu {
123416d8220SZaibo Xu if (req->c_req.encrypt)
124416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
125416d8220SZaibo Xu ctx->hlf_q_num;
126416d8220SZaibo Xu
127416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
128416d8220SZaibo Xu ctx->hlf_q_num;
129416d8220SZaibo Xu }
130416d8220SZaibo Xu
sec_free_queue_id(struct sec_ctx * ctx,struct sec_req * req)131a181647cSZaibo Xu static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
132416d8220SZaibo Xu {
133416d8220SZaibo Xu if (req->c_req.encrypt)
134416d8220SZaibo Xu atomic_dec(&ctx->enc_qcyclic);
135416d8220SZaibo Xu else
136416d8220SZaibo Xu atomic_dec(&ctx->dec_qcyclic);
137416d8220SZaibo Xu }
138416d8220SZaibo Xu
sec_alloc_req_id(struct sec_req * req,struct sec_qp_ctx * qp_ctx)139416d8220SZaibo Xu static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
140416d8220SZaibo Xu {
141416d8220SZaibo Xu int req_id;
142416d8220SZaibo Xu
14302884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
144129a9f34SWeili Qian req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);
14502884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
146b9c8d897SZaibo Xu if (unlikely(req_id < 0)) {
147a44dce50SLongfang Liu dev_err(req->ctx->dev, "alloc req id fail!\n");
148416d8220SZaibo Xu return req_id;
149416d8220SZaibo Xu }
150416d8220SZaibo Xu
151416d8220SZaibo Xu req->qp_ctx = qp_ctx;
152416d8220SZaibo Xu qp_ctx->req_list[req_id] = req;
153633e507fSLongfang Liu
154416d8220SZaibo Xu return req_id;
155416d8220SZaibo Xu }
156416d8220SZaibo Xu
sec_free_req_id(struct sec_req * req)157416d8220SZaibo Xu static void sec_free_req_id(struct sec_req *req)
158416d8220SZaibo Xu {
159416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
160416d8220SZaibo Xu int req_id = req->req_id;
161416d8220SZaibo Xu
162129a9f34SWeili Qian if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) {
163a44dce50SLongfang Liu dev_err(req->ctx->dev, "free request id invalid!\n");
164416d8220SZaibo Xu return;
165416d8220SZaibo Xu }
166416d8220SZaibo Xu
167416d8220SZaibo Xu qp_ctx->req_list[req_id] = NULL;
168416d8220SZaibo Xu req->qp_ctx = NULL;
169416d8220SZaibo Xu
17002884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
171416d8220SZaibo Xu idr_remove(&qp_ctx->req_idr, req_id);
17202884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
173416d8220SZaibo Xu }
174416d8220SZaibo Xu
pre_parse_finished_bd(struct bd_status * status,void * resp)175adc3f65aSKai Ye static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
176adc3f65aSKai Ye {
177adc3f65aSKai Ye struct sec_sqe *bd = resp;
178adc3f65aSKai Ye
179adc3f65aSKai Ye status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
180668f1ab7SKai Ye status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1;
181adc3f65aSKai Ye status->flag = (le16_to_cpu(bd->type2.done_flag) &
182adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
183adc3f65aSKai Ye status->tag = le16_to_cpu(bd->type2.tag);
184adc3f65aSKai Ye status->err_type = bd->type2.error_type;
185adc3f65aSKai Ye
186adc3f65aSKai Ye return bd->type_cipher_auth & SEC_TYPE_MASK;
187adc3f65aSKai Ye }
188adc3f65aSKai Ye
pre_parse_finished_bd3(struct bd_status * status,void * resp)189adc3f65aSKai Ye static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)
190adc3f65aSKai Ye {
191adc3f65aSKai Ye struct sec_sqe3 *bd3 = resp;
192adc3f65aSKai Ye
193adc3f65aSKai Ye status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK;
194668f1ab7SKai Ye status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1;
195adc3f65aSKai Ye status->flag = (le16_to_cpu(bd3->done_flag) &
196adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
197adc3f65aSKai Ye status->tag = le64_to_cpu(bd3->tag);
198adc3f65aSKai Ye status->err_type = bd3->error_type;
199adc3f65aSKai Ye
200adc3f65aSKai Ye return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK;
201adc3f65aSKai Ye }
202adc3f65aSKai Ye
sec_cb_status_check(struct sec_req * req,struct bd_status * status)203adc3f65aSKai Ye static int sec_cb_status_check(struct sec_req *req,
204adc3f65aSKai Ye struct bd_status *status)
205adc3f65aSKai Ye {
206adc3f65aSKai Ye struct sec_ctx *ctx = req->ctx;
207adc3f65aSKai Ye
208adc3f65aSKai Ye if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) {
209adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n",
210adc3f65aSKai Ye req->err_type, status->done);
211adc3f65aSKai Ye return -EIO;
212adc3f65aSKai Ye }
213adc3f65aSKai Ye
214adc3f65aSKai Ye if (unlikely(ctx->alg_type == SEC_SKCIPHER)) {
215adc3f65aSKai Ye if (unlikely(status->flag != SEC_SQE_CFLAG)) {
216adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "flag[%u]\n",
217adc3f65aSKai Ye status->flag);
218adc3f65aSKai Ye return -EIO;
219adc3f65aSKai Ye }
220668f1ab7SKai Ye } else if (unlikely(ctx->alg_type == SEC_AEAD)) {
221668f1ab7SKai Ye if (unlikely(status->flag != SEC_SQE_AEAD_FLAG ||
222668f1ab7SKai Ye status->icv == SEC_ICV_ERR)) {
223668f1ab7SKai Ye dev_err_ratelimited(ctx->dev,
224668f1ab7SKai Ye "flag[%u], icv[%u]\n",
225668f1ab7SKai Ye status->flag, status->icv);
226668f1ab7SKai Ye return -EBADMSG;
227668f1ab7SKai Ye }
228adc3f65aSKai Ye }
229adc3f65aSKai Ye
230adc3f65aSKai Ye return 0;
231adc3f65aSKai Ye }
232adc3f65aSKai Ye
sec_req_cb(struct hisi_qp * qp,void * resp)233416d8220SZaibo Xu static void sec_req_cb(struct hisi_qp *qp, void *resp)
234416d8220SZaibo Xu {
235416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
2368213a1a6SKai Ye struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;
237adc3f65aSKai Ye u8 type_supported = qp_ctx->ctx->type_supported;
238adc3f65aSKai Ye struct bd_status status;
239d6de2a59SZaibo Xu struct sec_ctx *ctx;
240d6de2a59SZaibo Xu struct sec_req *req;
241adc3f65aSKai Ye int err;
242416d8220SZaibo Xu u8 type;
243416d8220SZaibo Xu
244adc3f65aSKai Ye if (type_supported == SEC_BD_TYPE2) {
245adc3f65aSKai Ye type = pre_parse_finished_bd(&status, resp);
246adc3f65aSKai Ye req = qp_ctx->req_list[status.tag];
247adc3f65aSKai Ye } else {
248adc3f65aSKai Ye type = pre_parse_finished_bd3(&status, resp);
249adc3f65aSKai Ye req = (void *)(uintptr_t)status.tag;
250adc3f65aSKai Ye }
251adc3f65aSKai Ye
252adc3f65aSKai Ye if (unlikely(type != type_supported)) {
2538213a1a6SKai Ye atomic64_inc(&dfx->err_bd_cnt);
25449838259SKai Ye pr_err("err bd type [%u]\n", type);
255416d8220SZaibo Xu return;
256416d8220SZaibo Xu }
257416d8220SZaibo Xu
2588213a1a6SKai Ye if (unlikely(!req)) {
2598213a1a6SKai Ye atomic64_inc(&dfx->invalid_req_cnt);
2609597efc3SKai Ye atomic_inc(&qp->qp_status.used);
2618213a1a6SKai Ye return;
2628213a1a6SKai Ye }
263adc3f65aSKai Ye
264adc3f65aSKai Ye req->err_type = status.err_type;
265d6de2a59SZaibo Xu ctx = req->ctx;
266adc3f65aSKai Ye err = sec_cb_status_check(req, &status);
267adc3f65aSKai Ye if (err)
2688213a1a6SKai Ye atomic64_inc(&dfx->done_flag_cnt);
2691e9bc276SZaibo Xu
2708213a1a6SKai Ye atomic64_inc(&dfx->recv_cnt);
271416d8220SZaibo Xu
272d6de2a59SZaibo Xu ctx->req_op->buf_unmap(ctx, req);
273d6de2a59SZaibo Xu
274310ea0acSZaibo Xu ctx->req_op->callback(ctx, req, err);
275416d8220SZaibo Xu }
276416d8220SZaibo Xu
sec_bd_send(struct sec_ctx * ctx,struct sec_req * req)277416d8220SZaibo Xu static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
278416d8220SZaibo Xu {
279416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
280416d8220SZaibo Xu int ret;
281416d8220SZaibo Xu
2829597efc3SKai Ye if (ctx->fake_req_limit <=
2839597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) &&
2849597efc3SKai Ye !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))
2859597efc3SKai Ye return -EBUSY;
2869597efc3SKai Ye
28702884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
288416d8220SZaibo Xu ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
2899597efc3SKai Ye if (ctx->fake_req_limit <=
2909597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {
2919597efc3SKai Ye list_add_tail(&req->backlog_head, &qp_ctx->backlog);
292cb1eeb75SArnd Bergmann atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
2939597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
29402884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
2959597efc3SKai Ye return -EBUSY;
2969597efc3SKai Ye }
29702884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
298416d8220SZaibo Xu
299b9c8d897SZaibo Xu if (unlikely(ret == -EBUSY))
300416d8220SZaibo Xu return -ENOBUFS;
301416d8220SZaibo Xu
3029597efc3SKai Ye if (likely(!ret)) {
303416d8220SZaibo Xu ret = -EINPROGRESS;
3049597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
3058213a1a6SKai Ye }
306416d8220SZaibo Xu
307416d8220SZaibo Xu return ret;
308416d8220SZaibo Xu }
309416d8220SZaibo Xu
3107c7d902aSZaibo Xu /* Get DMA memory resources */
sec_alloc_civ_resource(struct device * dev,struct sec_alg_res * res)3117c7d902aSZaibo Xu static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
3127c7d902aSZaibo Xu {
313129a9f34SWeili Qian u16 q_depth = res->depth;
3147c7d902aSZaibo Xu int i;
3157c7d902aSZaibo Xu
316129a9f34SWeili Qian res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
3177c7d902aSZaibo Xu &res->c_ivin_dma, GFP_KERNEL);
3187c7d902aSZaibo Xu if (!res->c_ivin)
3197c7d902aSZaibo Xu return -ENOMEM;
3207c7d902aSZaibo Xu
321129a9f34SWeili Qian for (i = 1; i < q_depth; i++) {
3227c7d902aSZaibo Xu res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;
3237c7d902aSZaibo Xu res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;
3247c7d902aSZaibo Xu }
3257c7d902aSZaibo Xu
3267c7d902aSZaibo Xu return 0;
3277c7d902aSZaibo Xu }
3287c7d902aSZaibo Xu
sec_free_civ_resource(struct device * dev,struct sec_alg_res * res)3297c7d902aSZaibo Xu static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)
3307c7d902aSZaibo Xu {
3317c7d902aSZaibo Xu if (res->c_ivin)
332129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
3337c7d902aSZaibo Xu res->c_ivin, res->c_ivin_dma);
3347c7d902aSZaibo Xu }
3357c7d902aSZaibo Xu
sec_alloc_aiv_resource(struct device * dev,struct sec_alg_res * res)336c16a70c1SKai Ye static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res)
337c16a70c1SKai Ye {
338129a9f34SWeili Qian u16 q_depth = res->depth;
339c16a70c1SKai Ye int i;
340c16a70c1SKai Ye
341129a9f34SWeili Qian res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
342c16a70c1SKai Ye &res->a_ivin_dma, GFP_KERNEL);
343c16a70c1SKai Ye if (!res->a_ivin)
344c16a70c1SKai Ye return -ENOMEM;
345c16a70c1SKai Ye
346129a9f34SWeili Qian for (i = 1; i < q_depth; i++) {
347c16a70c1SKai Ye res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE;
348c16a70c1SKai Ye res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE;
349c16a70c1SKai Ye }
350c16a70c1SKai Ye
351c16a70c1SKai Ye return 0;
352c16a70c1SKai Ye }
353c16a70c1SKai Ye
sec_free_aiv_resource(struct device * dev,struct sec_alg_res * res)354c16a70c1SKai Ye static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res)
355c16a70c1SKai Ye {
356c16a70c1SKai Ye if (res->a_ivin)
357129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
358c16a70c1SKai Ye res->a_ivin, res->a_ivin_dma);
359c16a70c1SKai Ye }
360c16a70c1SKai Ye
sec_alloc_mac_resource(struct device * dev,struct sec_alg_res * res)3612f072d75SZaibo Xu static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)
3622f072d75SZaibo Xu {
363129a9f34SWeili Qian u16 q_depth = res->depth;
3642f072d75SZaibo Xu int i;
3652f072d75SZaibo Xu
366129a9f34SWeili Qian res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1,
3672f072d75SZaibo Xu &res->out_mac_dma, GFP_KERNEL);
3682f072d75SZaibo Xu if (!res->out_mac)
3692f072d75SZaibo Xu return -ENOMEM;
3702f072d75SZaibo Xu
371129a9f34SWeili Qian for (i = 1; i < q_depth; i++) {
3722f072d75SZaibo Xu res[i].out_mac_dma = res->out_mac_dma +
3732f072d75SZaibo Xu i * (SEC_MAX_MAC_LEN << 1);
3742f072d75SZaibo Xu res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);
3752f072d75SZaibo Xu }
3762f072d75SZaibo Xu
3772f072d75SZaibo Xu return 0;
3782f072d75SZaibo Xu }
3792f072d75SZaibo Xu
sec_free_mac_resource(struct device * dev,struct sec_alg_res * res)3802f072d75SZaibo Xu static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)
3812f072d75SZaibo Xu {
3822f072d75SZaibo Xu if (res->out_mac)
383129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1,
3842f072d75SZaibo Xu res->out_mac, res->out_mac_dma);
3852f072d75SZaibo Xu }
3862f072d75SZaibo Xu
sec_free_pbuf_resource(struct device * dev,struct sec_alg_res * res)38774b58db8SLongfang Liu static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res)
38874b58db8SLongfang Liu {
38974b58db8SLongfang Liu if (res->pbuf)
390129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth),
39174b58db8SLongfang Liu res->pbuf, res->pbuf_dma);
39274b58db8SLongfang Liu }
39374b58db8SLongfang Liu
39474b58db8SLongfang Liu /*
39574b58db8SLongfang Liu * To improve performance, pbuffer is used for
39674b58db8SLongfang Liu * small packets (< 512Bytes) as IOMMU translation using.
39774b58db8SLongfang Liu */
sec_alloc_pbuf_resource(struct device * dev,struct sec_alg_res * res)39874b58db8SLongfang Liu static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)
39974b58db8SLongfang Liu {
400129a9f34SWeili Qian u16 q_depth = res->depth;
401129a9f34SWeili Qian int size = SEC_PBUF_PAGE_NUM(q_depth);
40274b58db8SLongfang Liu int pbuf_page_offset;
40374b58db8SLongfang Liu int i, j, k;
40474b58db8SLongfang Liu
405129a9f34SWeili Qian res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth),
40674b58db8SLongfang Liu &res->pbuf_dma, GFP_KERNEL);
40774b58db8SLongfang Liu if (!res->pbuf)
40874b58db8SLongfang Liu return -ENOMEM;
40974b58db8SLongfang Liu
41074b58db8SLongfang Liu /*
41174b58db8SLongfang Liu * SEC_PBUF_PKG contains data pbuf, iv and
41274b58db8SLongfang Liu * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC>
41374b58db8SLongfang Liu * Every PAGE contains six SEC_PBUF_PKG
41474b58db8SLongfang Liu * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG
41574b58db8SLongfang Liu * So we need SEC_PBUF_PAGE_NUM numbers of PAGE
41674b58db8SLongfang Liu * for the SEC_TOTAL_PBUF_SZ
41774b58db8SLongfang Liu */
418129a9f34SWeili Qian for (i = 0; i <= size; i++) {
41974b58db8SLongfang Liu pbuf_page_offset = PAGE_SIZE * i;
42074b58db8SLongfang Liu for (j = 0; j < SEC_PBUF_NUM; j++) {
42174b58db8SLongfang Liu k = i * SEC_PBUF_NUM + j;
422129a9f34SWeili Qian if (k == q_depth)
42374b58db8SLongfang Liu break;
42474b58db8SLongfang Liu res[k].pbuf = res->pbuf +
42574b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset;
42674b58db8SLongfang Liu res[k].pbuf_dma = res->pbuf_dma +
42774b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset;
42874b58db8SLongfang Liu }
42974b58db8SLongfang Liu }
430633e507fSLongfang Liu
43174b58db8SLongfang Liu return 0;
43274b58db8SLongfang Liu }
43374b58db8SLongfang Liu
sec_alg_resource_alloc(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)4347c7d902aSZaibo Xu static int sec_alg_resource_alloc(struct sec_ctx *ctx,
4357c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx)
4367c7d902aSZaibo Xu {
4372f072d75SZaibo Xu struct sec_alg_res *res = qp_ctx->res;
438a44dce50SLongfang Liu struct device *dev = ctx->dev;
4392f072d75SZaibo Xu int ret;
4407c7d902aSZaibo Xu
4412f072d75SZaibo Xu ret = sec_alloc_civ_resource(dev, res);
4422f072d75SZaibo Xu if (ret)
4432f072d75SZaibo Xu return ret;
4442f072d75SZaibo Xu
4452f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) {
446c16a70c1SKai Ye ret = sec_alloc_aiv_resource(dev, res);
447c16a70c1SKai Ye if (ret)
448c16a70c1SKai Ye goto alloc_aiv_fail;
449c16a70c1SKai Ye
4502f072d75SZaibo Xu ret = sec_alloc_mac_resource(dev, res);
4512f072d75SZaibo Xu if (ret)
452c16a70c1SKai Ye goto alloc_mac_fail;
4532f072d75SZaibo Xu }
45474b58db8SLongfang Liu if (ctx->pbuf_supported) {
45574b58db8SLongfang Liu ret = sec_alloc_pbuf_resource(dev, res);
45674b58db8SLongfang Liu if (ret) {
45774b58db8SLongfang Liu dev_err(dev, "fail to alloc pbuf dma resource!\n");
45824efcec2SLongfang Liu goto alloc_pbuf_fail;
45974b58db8SLongfang Liu }
46074b58db8SLongfang Liu }
4612f072d75SZaibo Xu
4622f072d75SZaibo Xu return 0;
463633e507fSLongfang Liu
46424efcec2SLongfang Liu alloc_pbuf_fail:
46524efcec2SLongfang Liu if (ctx->alg_type == SEC_AEAD)
46624efcec2SLongfang Liu sec_free_mac_resource(dev, qp_ctx->res);
467c16a70c1SKai Ye alloc_mac_fail:
468c16a70c1SKai Ye if (ctx->alg_type == SEC_AEAD)
469c16a70c1SKai Ye sec_free_aiv_resource(dev, res);
470c16a70c1SKai Ye alloc_aiv_fail:
4712f072d75SZaibo Xu sec_free_civ_resource(dev, res);
4722f072d75SZaibo Xu return ret;
4737c7d902aSZaibo Xu }
4747c7d902aSZaibo Xu
sec_alg_resource_free(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)4757c7d902aSZaibo Xu static void sec_alg_resource_free(struct sec_ctx *ctx,
4767c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx)
4777c7d902aSZaibo Xu {
478a44dce50SLongfang Liu struct device *dev = ctx->dev;
4797c7d902aSZaibo Xu
4807c7d902aSZaibo Xu sec_free_civ_resource(dev, qp_ctx->res);
4812f072d75SZaibo Xu
48274b58db8SLongfang Liu if (ctx->pbuf_supported)
48374b58db8SLongfang Liu sec_free_pbuf_resource(dev, qp_ctx->res);
484*bba42507SChenghai Huang if (ctx->alg_type == SEC_AEAD) {
4852f072d75SZaibo Xu sec_free_mac_resource(dev, qp_ctx->res);
486*bba42507SChenghai Huang sec_free_aiv_resource(dev, qp_ctx->res);
487*bba42507SChenghai Huang }
4887c7d902aSZaibo Xu }
4897c7d902aSZaibo Xu
sec_alloc_qp_ctx_resource(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)4900c753f33SWenkai Lin static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
491129a9f34SWeili Qian {
492129a9f34SWeili Qian u16 q_depth = qp_ctx->qp->sq_depth;
493129a9f34SWeili Qian struct device *dev = ctx->dev;
494129a9f34SWeili Qian int ret = -ENOMEM;
495129a9f34SWeili Qian
496129a9f34SWeili Qian qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL);
497129a9f34SWeili Qian if (!qp_ctx->req_list)
498129a9f34SWeili Qian return ret;
499129a9f34SWeili Qian
500129a9f34SWeili Qian qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL);
501129a9f34SWeili Qian if (!qp_ctx->res)
502129a9f34SWeili Qian goto err_free_req_list;
503129a9f34SWeili Qian qp_ctx->res->depth = q_depth;
504129a9f34SWeili Qian
505129a9f34SWeili Qian qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
506129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_in_pool)) {
507129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for input!\n");
508129a9f34SWeili Qian goto err_free_res;
509129a9f34SWeili Qian }
510129a9f34SWeili Qian
511129a9f34SWeili Qian qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
512129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_out_pool)) {
513129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for output!\n");
514129a9f34SWeili Qian goto err_free_c_in_pool;
515129a9f34SWeili Qian }
516129a9f34SWeili Qian
517129a9f34SWeili Qian ret = sec_alg_resource_alloc(ctx, qp_ctx);
518129a9f34SWeili Qian if (ret)
519129a9f34SWeili Qian goto err_free_c_out_pool;
520129a9f34SWeili Qian
521129a9f34SWeili Qian return 0;
522129a9f34SWeili Qian
523129a9f34SWeili Qian err_free_c_out_pool:
524129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
525129a9f34SWeili Qian err_free_c_in_pool:
526129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
527129a9f34SWeili Qian err_free_res:
528129a9f34SWeili Qian kfree(qp_ctx->res);
529129a9f34SWeili Qian err_free_req_list:
530129a9f34SWeili Qian kfree(qp_ctx->req_list);
531129a9f34SWeili Qian return ret;
532129a9f34SWeili Qian }
533129a9f34SWeili Qian
sec_free_qp_ctx_resource(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)534129a9f34SWeili Qian static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
535129a9f34SWeili Qian {
536129a9f34SWeili Qian struct device *dev = ctx->dev;
537129a9f34SWeili Qian
538129a9f34SWeili Qian sec_alg_resource_free(ctx, qp_ctx);
539129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
540129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
541129a9f34SWeili Qian kfree(qp_ctx->res);
542129a9f34SWeili Qian kfree(qp_ctx->req_list);
543129a9f34SWeili Qian }
544129a9f34SWeili Qian
sec_create_qp_ctx(struct sec_ctx * ctx,int qp_ctx_id)5450c753f33SWenkai Lin static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id)
546416d8220SZaibo Xu {
547416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx;
548416d8220SZaibo Xu struct hisi_qp *qp;
549129a9f34SWeili Qian int ret;
550416d8220SZaibo Xu
551416d8220SZaibo Xu qp_ctx = &ctx->qp_ctx[qp_ctx_id];
5520b5e43bcSKai Ye qp = ctx->qps[qp_ctx_id];
553416d8220SZaibo Xu qp->req_type = 0;
554416d8220SZaibo Xu qp->qp_ctx = qp_ctx;
555416d8220SZaibo Xu qp_ctx->qp = qp;
556416d8220SZaibo Xu qp_ctx->ctx = ctx;
557416d8220SZaibo Xu
558adc3f65aSKai Ye qp->req_cb = sec_req_cb;
559adc3f65aSKai Ye
56002884a4fSZhengchao Shao spin_lock_init(&qp_ctx->req_lock);
561416d8220SZaibo Xu idr_init(&qp_ctx->req_idr);
5629597efc3SKai Ye INIT_LIST_HEAD(&qp_ctx->backlog);
563416d8220SZaibo Xu
5640c753f33SWenkai Lin ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx);
565416d8220SZaibo Xu if (ret)
566129a9f34SWeili Qian goto err_destroy_idr;
567416d8220SZaibo Xu
568416d8220SZaibo Xu ret = hisi_qm_start_qp(qp, 0);
569416d8220SZaibo Xu if (ret < 0)
570129a9f34SWeili Qian goto err_resource_free;
571416d8220SZaibo Xu
572416d8220SZaibo Xu return 0;
573416d8220SZaibo Xu
574129a9f34SWeili Qian err_resource_free:
575129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx);
576416d8220SZaibo Xu err_destroy_idr:
577416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr);
578416d8220SZaibo Xu return ret;
579416d8220SZaibo Xu }
580416d8220SZaibo Xu
sec_release_qp_ctx(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)581416d8220SZaibo Xu static void sec_release_qp_ctx(struct sec_ctx *ctx,
582416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx)
583416d8220SZaibo Xu {
584416d8220SZaibo Xu hisi_qm_stop_qp(qp_ctx->qp);
585129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx);
586416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr);
587416d8220SZaibo Xu }
588416d8220SZaibo Xu
sec_ctx_base_init(struct sec_ctx * ctx)589473a0f96SZaibo Xu static int sec_ctx_base_init(struct sec_ctx *ctx)
590416d8220SZaibo Xu {
591416d8220SZaibo Xu struct sec_dev *sec;
592416d8220SZaibo Xu int i, ret;
593416d8220SZaibo Xu
5940b5e43bcSKai Ye ctx->qps = sec_create_qps();
5950b5e43bcSKai Ye if (!ctx->qps) {
5960b5e43bcSKai Ye pr_err("Can not create sec qps!\n");
597416d8220SZaibo Xu return -ENODEV;
598416d8220SZaibo Xu }
5990b5e43bcSKai Ye
6000b5e43bcSKai Ye sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);
601416d8220SZaibo Xu ctx->sec = sec;
602a44dce50SLongfang Liu ctx->dev = &sec->qm.pdev->dev;
603a718cfceSZaibo Xu ctx->hlf_q_num = sec->ctx_q_num >> 1;
604416d8220SZaibo Xu
60574b58db8SLongfang Liu ctx->pbuf_supported = ctx->sec->iommu_used;
60674b58db8SLongfang Liu
607416d8220SZaibo Xu /* Half of queue depth is taken as fake requests limit in the queue. */
608129a9f34SWeili Qian ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1;
609416d8220SZaibo Xu ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
610416d8220SZaibo Xu GFP_KERNEL);
61124efcec2SLongfang Liu if (!ctx->qp_ctx) {
61224efcec2SLongfang Liu ret = -ENOMEM;
61324efcec2SLongfang Liu goto err_destroy_qps;
61424efcec2SLongfang Liu }
615416d8220SZaibo Xu
616416d8220SZaibo Xu for (i = 0; i < sec->ctx_q_num; i++) {
6170c753f33SWenkai Lin ret = sec_create_qp_ctx(ctx, i);
618416d8220SZaibo Xu if (ret)
619416d8220SZaibo Xu goto err_sec_release_qp_ctx;
620416d8220SZaibo Xu }
621416d8220SZaibo Xu
622416d8220SZaibo Xu return 0;
62324efcec2SLongfang Liu
624416d8220SZaibo Xu err_sec_release_qp_ctx:
625416d8220SZaibo Xu for (i = i - 1; i >= 0; i--)
626416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
627416d8220SZaibo Xu kfree(ctx->qp_ctx);
62824efcec2SLongfang Liu err_destroy_qps:
62924efcec2SLongfang Liu sec_destroy_qps(ctx->qps, sec->ctx_q_num);
630416d8220SZaibo Xu return ret;
631416d8220SZaibo Xu }
632416d8220SZaibo Xu
sec_ctx_base_uninit(struct sec_ctx * ctx)633473a0f96SZaibo Xu static void sec_ctx_base_uninit(struct sec_ctx *ctx)
634416d8220SZaibo Xu {
635473a0f96SZaibo Xu int i;
636416d8220SZaibo Xu
637416d8220SZaibo Xu for (i = 0; i < ctx->sec->ctx_q_num; i++)
638416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
639416d8220SZaibo Xu
6400b5e43bcSKai Ye sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num);
641416d8220SZaibo Xu kfree(ctx->qp_ctx);
642416d8220SZaibo Xu }
643416d8220SZaibo Xu
sec_cipher_init(struct sec_ctx * ctx)644473a0f96SZaibo Xu static int sec_cipher_init(struct sec_ctx *ctx)
645473a0f96SZaibo Xu {
646473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
647473a0f96SZaibo Xu
648a44dce50SLongfang Liu c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
649473a0f96SZaibo Xu &c_ctx->c_key_dma, GFP_KERNEL);
650473a0f96SZaibo Xu if (!c_ctx->c_key)
651473a0f96SZaibo Xu return -ENOMEM;
652473a0f96SZaibo Xu
653473a0f96SZaibo Xu return 0;
654473a0f96SZaibo Xu }
655473a0f96SZaibo Xu
sec_cipher_uninit(struct sec_ctx * ctx)656473a0f96SZaibo Xu static void sec_cipher_uninit(struct sec_ctx *ctx)
657473a0f96SZaibo Xu {
658473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
659473a0f96SZaibo Xu
660473a0f96SZaibo Xu memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
661a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
662473a0f96SZaibo Xu c_ctx->c_key, c_ctx->c_key_dma);
663473a0f96SZaibo Xu }
664473a0f96SZaibo Xu
sec_auth_init(struct sec_ctx * ctx)6652f072d75SZaibo Xu static int sec_auth_init(struct sec_ctx *ctx)
6662f072d75SZaibo Xu {
6672f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
6682f072d75SZaibo Xu
66945f5d017SKai Ye a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
6702f072d75SZaibo Xu &a_ctx->a_key_dma, GFP_KERNEL);
6712f072d75SZaibo Xu if (!a_ctx->a_key)
6722f072d75SZaibo Xu return -ENOMEM;
6732f072d75SZaibo Xu
6742f072d75SZaibo Xu return 0;
6752f072d75SZaibo Xu }
6762f072d75SZaibo Xu
sec_auth_uninit(struct sec_ctx * ctx)6772f072d75SZaibo Xu static void sec_auth_uninit(struct sec_ctx *ctx)
6782f072d75SZaibo Xu {
6792f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
6802f072d75SZaibo Xu
68145f5d017SKai Ye memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE);
68245f5d017SKai Ye dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
6832f072d75SZaibo Xu a_ctx->a_key, a_ctx->a_key_dma);
6842f072d75SZaibo Xu }
6852f072d75SZaibo Xu
sec_skcipher_fbtfm_init(struct crypto_skcipher * tfm)6865652d55aSKai Ye static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm)
6875652d55aSKai Ye {
6885652d55aSKai Ye const char *alg = crypto_tfm_alg_name(&tfm->base);
6895652d55aSKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
6905652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
6915652d55aSKai Ye
6925652d55aSKai Ye c_ctx->fallback = false;
693e764d81dSKai Ye
694e764d81dSKai Ye /* Currently, only XTS mode need fallback tfm when using 192bit key */
6955652d55aSKai Ye if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ)))
6965652d55aSKai Ye return 0;
6975652d55aSKai Ye
6985652d55aSKai Ye c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0,
6995652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK);
7005652d55aSKai Ye if (IS_ERR(c_ctx->fbtfm)) {
701e764d81dSKai Ye pr_err("failed to alloc xts mode fallback tfm!\n");
7025652d55aSKai Ye return PTR_ERR(c_ctx->fbtfm);
7035652d55aSKai Ye }
7045652d55aSKai Ye
7055652d55aSKai Ye return 0;
7065652d55aSKai Ye }
7075652d55aSKai Ye
sec_skcipher_init(struct crypto_skcipher * tfm)708473a0f96SZaibo Xu static int sec_skcipher_init(struct crypto_skcipher *tfm)
709473a0f96SZaibo Xu {
710473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
711473a0f96SZaibo Xu int ret;
712473a0f96SZaibo Xu
7132f072d75SZaibo Xu ctx->alg_type = SEC_SKCIPHER;
714473a0f96SZaibo Xu crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
715473a0f96SZaibo Xu ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
716473a0f96SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
7174b7aef02SLongfang Liu pr_err("get error skcipher iv size!\n");
718473a0f96SZaibo Xu return -EINVAL;
719473a0f96SZaibo Xu }
720473a0f96SZaibo Xu
721473a0f96SZaibo Xu ret = sec_ctx_base_init(ctx);
722473a0f96SZaibo Xu if (ret)
723473a0f96SZaibo Xu return ret;
724473a0f96SZaibo Xu
725473a0f96SZaibo Xu ret = sec_cipher_init(ctx);
726473a0f96SZaibo Xu if (ret)
727473a0f96SZaibo Xu goto err_cipher_init;
728473a0f96SZaibo Xu
7295652d55aSKai Ye ret = sec_skcipher_fbtfm_init(tfm);
7305652d55aSKai Ye if (ret)
7315652d55aSKai Ye goto err_fbtfm_init;
7325652d55aSKai Ye
733473a0f96SZaibo Xu return 0;
734633e507fSLongfang Liu
7355652d55aSKai Ye err_fbtfm_init:
7365652d55aSKai Ye sec_cipher_uninit(ctx);
737473a0f96SZaibo Xu err_cipher_init:
738473a0f96SZaibo Xu sec_ctx_base_uninit(ctx);
739473a0f96SZaibo Xu return ret;
740473a0f96SZaibo Xu }
741473a0f96SZaibo Xu
sec_skcipher_uninit(struct crypto_skcipher * tfm)742473a0f96SZaibo Xu static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
743473a0f96SZaibo Xu {
744473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
745473a0f96SZaibo Xu
7465652d55aSKai Ye if (ctx->c_ctx.fbtfm)
7475652d55aSKai Ye crypto_free_sync_skcipher(ctx->c_ctx.fbtfm);
7485652d55aSKai Ye
749473a0f96SZaibo Xu sec_cipher_uninit(ctx);
750473a0f96SZaibo Xu sec_ctx_base_uninit(ctx);
751473a0f96SZaibo Xu }
752473a0f96SZaibo Xu
sec_skcipher_3des_setkey(struct crypto_skcipher * tfm,const u8 * key,const u32 keylen)7530c753f33SWenkai Lin static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen)
754416d8220SZaibo Xu {
755ae6ce7b1SKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
756ae6ce7b1SKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
757ae6ce7b1SKai Ye int ret;
758ae6ce7b1SKai Ye
759ae6ce7b1SKai Ye ret = verify_skcipher_des3_key(tfm, key);
760ae6ce7b1SKai Ye if (ret)
761ae6ce7b1SKai Ye return ret;
762ae6ce7b1SKai Ye
763416d8220SZaibo Xu switch (keylen) {
764416d8220SZaibo Xu case SEC_DES3_2KEY_SIZE:
765416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
766416d8220SZaibo Xu break;
767416d8220SZaibo Xu case SEC_DES3_3KEY_SIZE:
768416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;
769416d8220SZaibo Xu break;
770416d8220SZaibo Xu default:
771416d8220SZaibo Xu return -EINVAL;
772416d8220SZaibo Xu }
773416d8220SZaibo Xu
774416d8220SZaibo Xu return 0;
775416d8220SZaibo Xu }
776416d8220SZaibo Xu
sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx * c_ctx,const u32 keylen,const enum sec_cmode c_mode)777416d8220SZaibo Xu static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,
778416d8220SZaibo Xu const u32 keylen,
779416d8220SZaibo Xu const enum sec_cmode c_mode)
780416d8220SZaibo Xu {
781416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) {
782416d8220SZaibo Xu switch (keylen) {
783416d8220SZaibo Xu case SEC_XTS_MIN_KEY_SIZE:
784416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT;
785416d8220SZaibo Xu break;
7865652d55aSKai Ye case SEC_XTS_MID_KEY_SIZE:
7875652d55aSKai Ye c_ctx->fallback = true;
7885652d55aSKai Ye break;
789416d8220SZaibo Xu case SEC_XTS_MAX_KEY_SIZE:
790416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT;
791416d8220SZaibo Xu break;
792416d8220SZaibo Xu default:
793416d8220SZaibo Xu pr_err("hisi_sec2: xts mode key error!\n");
794416d8220SZaibo Xu return -EINVAL;
795416d8220SZaibo Xu }
796416d8220SZaibo Xu } else {
797adc3f65aSKai Ye if (c_ctx->c_alg == SEC_CALG_SM4 &&
798adc3f65aSKai Ye keylen != AES_KEYSIZE_128) {
799adc3f65aSKai Ye pr_err("hisi_sec2: sm4 key error!\n");
800adc3f65aSKai Ye return -EINVAL;
801adc3f65aSKai Ye } else {
802416d8220SZaibo Xu switch (keylen) {
803416d8220SZaibo Xu case AES_KEYSIZE_128:
804416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT;
805416d8220SZaibo Xu break;
806416d8220SZaibo Xu case AES_KEYSIZE_192:
807416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT;
808416d8220SZaibo Xu break;
809416d8220SZaibo Xu case AES_KEYSIZE_256:
810416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT;
811416d8220SZaibo Xu break;
812416d8220SZaibo Xu default:
813416d8220SZaibo Xu pr_err("hisi_sec2: aes key error!\n");
814416d8220SZaibo Xu return -EINVAL;
815416d8220SZaibo Xu }
816416d8220SZaibo Xu }
817adc3f65aSKai Ye }
818416d8220SZaibo Xu
819416d8220SZaibo Xu return 0;
820416d8220SZaibo Xu }
821416d8220SZaibo Xu
sec_skcipher_setkey(struct crypto_skcipher * tfm,const u8 * key,const u32 keylen,const enum sec_calg c_alg,const enum sec_cmode c_mode)822416d8220SZaibo Xu static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
823416d8220SZaibo Xu const u32 keylen, const enum sec_calg c_alg,
824416d8220SZaibo Xu const enum sec_cmode c_mode)
825416d8220SZaibo Xu {
826416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
827416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
828a44dce50SLongfang Liu struct device *dev = ctx->dev;
829416d8220SZaibo Xu int ret;
830416d8220SZaibo Xu
831416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) {
832416d8220SZaibo Xu ret = xts_verify_key(tfm, key, keylen);
833416d8220SZaibo Xu if (ret) {
834a44dce50SLongfang Liu dev_err(dev, "xts mode key err!\n");
835416d8220SZaibo Xu return ret;
836416d8220SZaibo Xu }
837416d8220SZaibo Xu }
838416d8220SZaibo Xu
839416d8220SZaibo Xu c_ctx->c_alg = c_alg;
840416d8220SZaibo Xu c_ctx->c_mode = c_mode;
841416d8220SZaibo Xu
842416d8220SZaibo Xu switch (c_alg) {
843416d8220SZaibo Xu case SEC_CALG_3DES:
8440c753f33SWenkai Lin ret = sec_skcipher_3des_setkey(tfm, key, keylen);
845416d8220SZaibo Xu break;
846416d8220SZaibo Xu case SEC_CALG_AES:
847416d8220SZaibo Xu case SEC_CALG_SM4:
848416d8220SZaibo Xu ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
849416d8220SZaibo Xu break;
850416d8220SZaibo Xu default:
8511bed8225SChenghai Huang dev_err(dev, "sec c_alg err!\n");
852416d8220SZaibo Xu return -EINVAL;
853416d8220SZaibo Xu }
854416d8220SZaibo Xu
855416d8220SZaibo Xu if (ret) {
856a44dce50SLongfang Liu dev_err(dev, "set sec key err!\n");
857416d8220SZaibo Xu return ret;
858416d8220SZaibo Xu }
859416d8220SZaibo Xu
860416d8220SZaibo Xu memcpy(c_ctx->c_key, key, keylen);
861e764d81dSKai Ye if (c_ctx->fallback && c_ctx->fbtfm) {
8625652d55aSKai Ye ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen);
8635652d55aSKai Ye if (ret) {
8645652d55aSKai Ye dev_err(dev, "failed to set fallback skcipher key!\n");
8655652d55aSKai Ye return ret;
8665652d55aSKai Ye }
8675652d55aSKai Ye }
868416d8220SZaibo Xu return 0;
869416d8220SZaibo Xu }
870416d8220SZaibo Xu
871416d8220SZaibo Xu #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \
872416d8220SZaibo Xu static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\
873416d8220SZaibo Xu u32 keylen) \
874416d8220SZaibo Xu { \
875416d8220SZaibo Xu return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \
876416d8220SZaibo Xu }
877416d8220SZaibo Xu
GEN_SEC_SETKEY_FUNC(aes_ecb,SEC_CALG_AES,SEC_CMODE_ECB)878416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)
879416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)
880416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)
8817b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR)
882416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)
883416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)
884416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)
885416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)
8867b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR)
887416d8220SZaibo Xu
88874b58db8SLongfang Liu static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
88974b58db8SLongfang Liu struct scatterlist *src)
89074b58db8SLongfang Liu {
8916c46a329SKai Ye struct sec_aead_req *a_req = &req->aead_req;
8926c46a329SKai Ye struct aead_request *aead_req = a_req->aead_req;
89374b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
89474b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
895a44dce50SLongfang Liu struct device *dev = ctx->dev;
89674b58db8SLongfang Liu int copy_size, pbuf_length;
89774b58db8SLongfang Liu int req_id = req->req_id;
8986c46a329SKai Ye struct crypto_aead *tfm;
8996c46a329SKai Ye size_t authsize;
9006c46a329SKai Ye u8 *mac_offset;
90174b58db8SLongfang Liu
90274b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD)
90374b58db8SLongfang Liu copy_size = aead_req->cryptlen + aead_req->assoclen;
90474b58db8SLongfang Liu else
90574b58db8SLongfang Liu copy_size = c_req->c_len;
90674b58db8SLongfang Liu
90774b58db8SLongfang Liu pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
9086c46a329SKai Ye qp_ctx->res[req_id].pbuf, copy_size);
90974b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) {
91074b58db8SLongfang Liu dev_err(dev, "copy src data to pbuf error!\n");
91174b58db8SLongfang Liu return -EINVAL;
91274b58db8SLongfang Liu }
9136c46a329SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
9146c46a329SKai Ye tfm = crypto_aead_reqtfm(aead_req);
9156c46a329SKai Ye authsize = crypto_aead_authsize(tfm);
9166c46a329SKai Ye mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;
9176c46a329SKai Ye memcpy(a_req->out_mac, mac_offset, authsize);
9186c46a329SKai Ye }
91974b58db8SLongfang Liu
9209039878aSKai Ye req->in_dma = qp_ctx->res[req_id].pbuf_dma;
9219039878aSKai Ye c_req->c_out_dma = req->in_dma;
92274b58db8SLongfang Liu
92374b58db8SLongfang Liu return 0;
92474b58db8SLongfang Liu }
92574b58db8SLongfang Liu
sec_cipher_pbuf_unmap(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * dst)92674b58db8SLongfang Liu static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
92774b58db8SLongfang Liu struct scatterlist *dst)
92874b58db8SLongfang Liu {
92974b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req;
93074b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
93174b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
93274b58db8SLongfang Liu int copy_size, pbuf_length;
93374b58db8SLongfang Liu int req_id = req->req_id;
93474b58db8SLongfang Liu
93574b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD)
93674b58db8SLongfang Liu copy_size = c_req->c_len + aead_req->assoclen;
93774b58db8SLongfang Liu else
93874b58db8SLongfang Liu copy_size = c_req->c_len;
93974b58db8SLongfang Liu
94074b58db8SLongfang Liu pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
941668f1ab7SKai Ye qp_ctx->res[req_id].pbuf, copy_size);
94274b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size))
943668f1ab7SKai Ye dev_err(ctx->dev, "copy pbuf data to dst error!\n");
944668f1ab7SKai Ye }
945668f1ab7SKai Ye
sec_aead_mac_init(struct sec_aead_req * req)946668f1ab7SKai Ye static int sec_aead_mac_init(struct sec_aead_req *req)
947668f1ab7SKai Ye {
948668f1ab7SKai Ye struct aead_request *aead_req = req->aead_req;
949668f1ab7SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
950668f1ab7SKai Ye size_t authsize = crypto_aead_authsize(tfm);
951668f1ab7SKai Ye u8 *mac_out = req->out_mac;
952668f1ab7SKai Ye struct scatterlist *sgl = aead_req->src;
953668f1ab7SKai Ye size_t copy_size;
954668f1ab7SKai Ye off_t skip_size;
955668f1ab7SKai Ye
956668f1ab7SKai Ye /* Copy input mac */
957668f1ab7SKai Ye skip_size = aead_req->assoclen + aead_req->cryptlen - authsize;
958668f1ab7SKai Ye copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out,
959668f1ab7SKai Ye authsize, skip_size);
960668f1ab7SKai Ye if (unlikely(copy_size != authsize))
961668f1ab7SKai Ye return -EINVAL;
962668f1ab7SKai Ye
963668f1ab7SKai Ye return 0;
96474b58db8SLongfang Liu }
96574b58db8SLongfang Liu
sec_cipher_map(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)9662514f559SLongfang Liu static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
967416d8220SZaibo Xu struct scatterlist *src, struct scatterlist *dst)
968416d8220SZaibo Xu {
969416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
9702514f559SLongfang Liu struct sec_aead_req *a_req = &req->aead_req;
971416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
9722514f559SLongfang Liu struct sec_alg_res *res = &qp_ctx->res[req->req_id];
973a44dce50SLongfang Liu struct device *dev = ctx->dev;
97474b58db8SLongfang Liu int ret;
9752514f559SLongfang Liu
97674b58db8SLongfang Liu if (req->use_pbuf) {
97774b58db8SLongfang Liu c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET;
97874b58db8SLongfang Liu c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET;
97974b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) {
980c16a70c1SKai Ye a_req->a_ivin = res->a_ivin;
981c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma;
98274b58db8SLongfang Liu a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET;
98374b58db8SLongfang Liu a_req->out_mac_dma = res->pbuf_dma +
98474b58db8SLongfang Liu SEC_PBUF_MAC_OFFSET;
98574b58db8SLongfang Liu }
986668f1ab7SKai Ye ret = sec_cipher_pbuf_map(ctx, req, src);
98774b58db8SLongfang Liu
98874b58db8SLongfang Liu return ret;
98974b58db8SLongfang Liu }
9902514f559SLongfang Liu c_req->c_ivin = res->c_ivin;
9912514f559SLongfang Liu c_req->c_ivin_dma = res->c_ivin_dma;
9922514f559SLongfang Liu if (ctx->alg_type == SEC_AEAD) {
993c16a70c1SKai Ye a_req->a_ivin = res->a_ivin;
994c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma;
9952514f559SLongfang Liu a_req->out_mac = res->out_mac;
9962514f559SLongfang Liu a_req->out_mac_dma = res->out_mac_dma;
9972514f559SLongfang Liu }
998416d8220SZaibo Xu
9999039878aSKai Ye req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
1000416d8220SZaibo Xu qp_ctx->c_in_pool,
1001416d8220SZaibo Xu req->req_id,
10029039878aSKai Ye &req->in_dma);
10039039878aSKai Ye if (IS_ERR(req->in)) {
1004416d8220SZaibo Xu dev_err(dev, "fail to dma map input sgl buffers!\n");
10059039878aSKai Ye return PTR_ERR(req->in);
1006416d8220SZaibo Xu }
1007416d8220SZaibo Xu
1008668f1ab7SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
1009668f1ab7SKai Ye ret = sec_aead_mac_init(a_req);
1010668f1ab7SKai Ye if (unlikely(ret)) {
1011668f1ab7SKai Ye dev_err(dev, "fail to init mac data for ICV!\n");
1012ce2cb2e1SWenkai Lin hisi_acc_sg_buf_unmap(dev, src, req->in);
1013668f1ab7SKai Ye return ret;
1014668f1ab7SKai Ye }
1015668f1ab7SKai Ye }
10169039878aSKai Ye
1017416d8220SZaibo Xu if (dst == src) {
10189039878aSKai Ye c_req->c_out = req->in;
10199039878aSKai Ye c_req->c_out_dma = req->in_dma;
1020416d8220SZaibo Xu } else {
1021416d8220SZaibo Xu c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
1022416d8220SZaibo Xu qp_ctx->c_out_pool,
1023416d8220SZaibo Xu req->req_id,
1024416d8220SZaibo Xu &c_req->c_out_dma);
1025416d8220SZaibo Xu
1026416d8220SZaibo Xu if (IS_ERR(c_req->c_out)) {
1027416d8220SZaibo Xu dev_err(dev, "fail to dma map output sgl buffers!\n");
10289039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in);
1029416d8220SZaibo Xu return PTR_ERR(c_req->c_out);
1030416d8220SZaibo Xu }
1031416d8220SZaibo Xu }
1032416d8220SZaibo Xu
1033416d8220SZaibo Xu return 0;
1034416d8220SZaibo Xu }
1035416d8220SZaibo Xu
sec_cipher_unmap(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)10362514f559SLongfang Liu static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
1037a181647cSZaibo Xu struct scatterlist *src, struct scatterlist *dst)
1038a181647cSZaibo Xu {
10392514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1040a44dce50SLongfang Liu struct device *dev = ctx->dev;
1041a181647cSZaibo Xu
104274b58db8SLongfang Liu if (req->use_pbuf) {
104374b58db8SLongfang Liu sec_cipher_pbuf_unmap(ctx, req, dst);
104474b58db8SLongfang Liu } else {
10452514f559SLongfang Liu if (dst != src)
10469039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in);
10472514f559SLongfang Liu
10482514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out);
1049a181647cSZaibo Xu }
105074b58db8SLongfang Liu }
1051a181647cSZaibo Xu
sec_skcipher_sgl_map(struct sec_ctx * ctx,struct sec_req * req)1052416d8220SZaibo Xu static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1053416d8220SZaibo Xu {
1054a181647cSZaibo Xu struct skcipher_request *sq = req->c_req.sk_req;
1055416d8220SZaibo Xu
10562514f559SLongfang Liu return sec_cipher_map(ctx, req, sq->src, sq->dst);
1057416d8220SZaibo Xu }
1058416d8220SZaibo Xu
sec_skcipher_sgl_unmap(struct sec_ctx * ctx,struct sec_req * req)1059416d8220SZaibo Xu static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1060416d8220SZaibo Xu {
10612514f559SLongfang Liu struct skcipher_request *sq = req->c_req.sk_req;
1062416d8220SZaibo Xu
10632514f559SLongfang Liu sec_cipher_unmap(ctx, req, sq->src, sq->dst);
1064416d8220SZaibo Xu }
1065416d8220SZaibo Xu
sec_aead_aes_set_key(struct sec_cipher_ctx * c_ctx,struct crypto_authenc_keys * keys)10662f072d75SZaibo Xu static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,
10672f072d75SZaibo Xu struct crypto_authenc_keys *keys)
10682f072d75SZaibo Xu {
10692f072d75SZaibo Xu switch (keys->enckeylen) {
10702f072d75SZaibo Xu case AES_KEYSIZE_128:
10712f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT;
10722f072d75SZaibo Xu break;
10732f072d75SZaibo Xu case AES_KEYSIZE_192:
10742f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT;
10752f072d75SZaibo Xu break;
10762f072d75SZaibo Xu case AES_KEYSIZE_256:
10772f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT;
10782f072d75SZaibo Xu break;
10792f072d75SZaibo Xu default:
10802f072d75SZaibo Xu pr_err("hisi_sec2: aead aes key error!\n");
10812f072d75SZaibo Xu return -EINVAL;
10822f072d75SZaibo Xu }
10832f072d75SZaibo Xu memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);
10842f072d75SZaibo Xu
10852f072d75SZaibo Xu return 0;
10862f072d75SZaibo Xu }
10872f072d75SZaibo Xu
sec_aead_auth_set_key(struct sec_auth_ctx * ctx,struct crypto_authenc_keys * keys)10882f072d75SZaibo Xu static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
10892f072d75SZaibo Xu struct crypto_authenc_keys *keys)
10902f072d75SZaibo Xu {
10912f072d75SZaibo Xu struct crypto_shash *hash_tfm = ctx->hash_tfm;
10925761498cSKai Ye int blocksize, digestsize, ret;
10932f072d75SZaibo Xu
10942f072d75SZaibo Xu if (!keys->authkeylen) {
10952f072d75SZaibo Xu pr_err("hisi_sec2: aead auth key error!\n");
10962f072d75SZaibo Xu return -EINVAL;
10972f072d75SZaibo Xu }
10982f072d75SZaibo Xu
10992f072d75SZaibo Xu blocksize = crypto_shash_blocksize(hash_tfm);
11005761498cSKai Ye digestsize = crypto_shash_digestsize(hash_tfm);
11012f072d75SZaibo Xu if (keys->authkeylen > blocksize) {
110261c38e3aSEric Biggers ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,
11032f072d75SZaibo Xu keys->authkeylen, ctx->a_key);
11042f072d75SZaibo Xu if (ret) {
11052203d3f7SColin Ian King pr_err("hisi_sec2: aead auth digest error!\n");
11062f072d75SZaibo Xu return -EINVAL;
11072f072d75SZaibo Xu }
11085761498cSKai Ye ctx->a_key_len = digestsize;
11092f072d75SZaibo Xu } else {
11102f072d75SZaibo Xu memcpy(ctx->a_key, keys->authkey, keys->authkeylen);
11112f072d75SZaibo Xu ctx->a_key_len = keys->authkeylen;
11122f072d75SZaibo Xu }
11132f072d75SZaibo Xu
11142f072d75SZaibo Xu return 0;
11152f072d75SZaibo Xu }
11162f072d75SZaibo Xu
sec_aead_setauthsize(struct crypto_aead * aead,unsigned int authsize)11176c46a329SKai Ye static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize)
11186c46a329SKai Ye {
11196c46a329SKai Ye struct crypto_tfm *tfm = crypto_aead_tfm(aead);
11206c46a329SKai Ye struct sec_ctx *ctx = crypto_tfm_ctx(tfm);
11216c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
11226c46a329SKai Ye
11236c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm))
11246c46a329SKai Ye return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize);
11256c46a329SKai Ye
11266c46a329SKai Ye return 0;
11276c46a329SKai Ye }
11286c46a329SKai Ye
sec_aead_fallback_setkey(struct sec_auth_ctx * a_ctx,struct crypto_aead * tfm,const u8 * key,unsigned int keylen)11296c46a329SKai Ye static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx,
11306c46a329SKai Ye struct crypto_aead *tfm, const u8 *key,
11316c46a329SKai Ye unsigned int keylen)
11326c46a329SKai Ye {
11336c46a329SKai Ye crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK);
11346c46a329SKai Ye crypto_aead_set_flags(a_ctx->fallback_aead_tfm,
11356c46a329SKai Ye crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
11366c46a329SKai Ye return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen);
11376c46a329SKai Ye }
11386c46a329SKai Ye
sec_aead_setkey(struct crypto_aead * tfm,const u8 * key,const u32 keylen,const enum sec_hash_alg a_alg,const enum sec_calg c_alg,const enum sec_mac_len mac_len,const enum sec_cmode c_mode)11392f072d75SZaibo Xu static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
11402f072d75SZaibo Xu const u32 keylen, const enum sec_hash_alg a_alg,
11412f072d75SZaibo Xu const enum sec_calg c_alg,
11422f072d75SZaibo Xu const enum sec_mac_len mac_len,
11432f072d75SZaibo Xu const enum sec_cmode c_mode)
11442f072d75SZaibo Xu {
11452f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
11462f072d75SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
11476c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1148a44dce50SLongfang Liu struct device *dev = ctx->dev;
11492f072d75SZaibo Xu struct crypto_authenc_keys keys;
11502f072d75SZaibo Xu int ret;
11512f072d75SZaibo Xu
11522f072d75SZaibo Xu ctx->a_ctx.a_alg = a_alg;
11532f072d75SZaibo Xu ctx->c_ctx.c_alg = c_alg;
11542f072d75SZaibo Xu ctx->a_ctx.mac_len = mac_len;
11552f072d75SZaibo Xu c_ctx->c_mode = c_mode;
11562f072d75SZaibo Xu
1157c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) {
1158c16a70c1SKai Ye ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
1159c16a70c1SKai Ye if (ret) {
1160c16a70c1SKai Ye dev_err(dev, "set sec aes ccm cipher key err!\n");
1161c16a70c1SKai Ye return ret;
1162c16a70c1SKai Ye }
1163c16a70c1SKai Ye memcpy(c_ctx->c_key, key, keylen);
1164c16a70c1SKai Ye
11656c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) {
11666c46a329SKai Ye ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
11676c46a329SKai Ye if (ret)
11686c46a329SKai Ye return ret;
11696c46a329SKai Ye }
11706c46a329SKai Ye
1171c16a70c1SKai Ye return 0;
1172c16a70c1SKai Ye }
1173c16a70c1SKai Ye
11741bed8225SChenghai Huang ret = crypto_authenc_extractkeys(&keys, key, keylen);
11751bed8225SChenghai Huang if (ret)
11762f072d75SZaibo Xu goto bad_key;
11772f072d75SZaibo Xu
11782f072d75SZaibo Xu ret = sec_aead_aes_set_key(c_ctx, &keys);
11792f072d75SZaibo Xu if (ret) {
1180a44dce50SLongfang Liu dev_err(dev, "set sec cipher key err!\n");
11812f072d75SZaibo Xu goto bad_key;
11822f072d75SZaibo Xu }
11832f072d75SZaibo Xu
11842f072d75SZaibo Xu ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
11852f072d75SZaibo Xu if (ret) {
1186a44dce50SLongfang Liu dev_err(dev, "set sec auth key err!\n");
11872f072d75SZaibo Xu goto bad_key;
11882f072d75SZaibo Xu }
11892f072d75SZaibo Xu
1190adc3f65aSKai Ye if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) ||
1191adc3f65aSKai Ye (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) {
11921bed8225SChenghai Huang ret = -EINVAL;
1193adc3f65aSKai Ye dev_err(dev, "MAC or AUTH key length error!\n");
1194adc3f65aSKai Ye goto bad_key;
1195adc3f65aSKai Ye }
1196adc3f65aSKai Ye
11972f072d75SZaibo Xu return 0;
1198633e507fSLongfang Liu
11992f072d75SZaibo Xu bad_key:
12002f072d75SZaibo Xu memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));
12011bed8225SChenghai Huang return ret;
12022f072d75SZaibo Xu }
12032f072d75SZaibo Xu
12042f072d75SZaibo Xu
12052f072d75SZaibo Xu #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \
12062f072d75SZaibo Xu static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \
12072f072d75SZaibo Xu u32 keylen) \
12082f072d75SZaibo Xu { \
12092f072d75SZaibo Xu return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\
12102f072d75SZaibo Xu }
12112f072d75SZaibo Xu
GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1,SEC_A_HMAC_SHA1,SEC_CALG_AES,SEC_HMAC_SHA1_MAC,SEC_CMODE_CBC)12122f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1,
12132f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC)
12142f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256,
12152f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC)
12162f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512,
12172f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC)
1218c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES,
1219c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1220c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES,
1221c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
1222c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4,
1223c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1224c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4,
1225c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
12262f072d75SZaibo Xu
12272f072d75SZaibo Xu static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
12282f072d75SZaibo Xu {
12292f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req;
12302f072d75SZaibo Xu
12312514f559SLongfang Liu return sec_cipher_map(ctx, req, aq->src, aq->dst);
12322f072d75SZaibo Xu }
12332f072d75SZaibo Xu
sec_aead_sgl_unmap(struct sec_ctx * ctx,struct sec_req * req)12342f072d75SZaibo Xu static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
12352f072d75SZaibo Xu {
12362f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req;
12372f072d75SZaibo Xu
12382514f559SLongfang Liu sec_cipher_unmap(ctx, req, aq->src, aq->dst);
12392f072d75SZaibo Xu }
12402f072d75SZaibo Xu
sec_request_transfer(struct sec_ctx * ctx,struct sec_req * req)1241416d8220SZaibo Xu static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)
1242416d8220SZaibo Xu {
1243416d8220SZaibo Xu int ret;
1244416d8220SZaibo Xu
1245416d8220SZaibo Xu ret = ctx->req_op->buf_map(ctx, req);
1246b9c8d897SZaibo Xu if (unlikely(ret))
1247416d8220SZaibo Xu return ret;
1248416d8220SZaibo Xu
1249416d8220SZaibo Xu ctx->req_op->do_transfer(ctx, req);
1250416d8220SZaibo Xu
1251416d8220SZaibo Xu ret = ctx->req_op->bd_fill(ctx, req);
1252b9c8d897SZaibo Xu if (unlikely(ret))
1253416d8220SZaibo Xu goto unmap_req_buf;
1254416d8220SZaibo Xu
1255416d8220SZaibo Xu return ret;
1256416d8220SZaibo Xu
1257416d8220SZaibo Xu unmap_req_buf:
1258416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req);
1259416d8220SZaibo Xu return ret;
1260416d8220SZaibo Xu }
1261416d8220SZaibo Xu
sec_request_untransfer(struct sec_ctx * ctx,struct sec_req * req)1262416d8220SZaibo Xu static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)
1263416d8220SZaibo Xu {
1264416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req);
1265416d8220SZaibo Xu }
1266416d8220SZaibo Xu
sec_skcipher_copy_iv(struct sec_ctx * ctx,struct sec_req * req)1267416d8220SZaibo Xu static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
1268416d8220SZaibo Xu {
1269416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req;
12702514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1271416d8220SZaibo Xu
12722514f559SLongfang Liu memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize);
1273416d8220SZaibo Xu }
1274416d8220SZaibo Xu
sec_skcipher_bd_fill(struct sec_ctx * ctx,struct sec_req * req)1275416d8220SZaibo Xu static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1276416d8220SZaibo Xu {
1277416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1278416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
1279416d8220SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe;
1280416d8220SZaibo Xu u8 scene, sa_type, da_type;
1281416d8220SZaibo Xu u8 bd_type, cipher;
12827c7d902aSZaibo Xu u8 de = 0;
1283416d8220SZaibo Xu
1284416d8220SZaibo Xu memset(sec_sqe, 0, sizeof(struct sec_sqe));
1285416d8220SZaibo Xu
1286416d8220SZaibo Xu sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
12872514f559SLongfang Liu sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
12889039878aSKai Ye sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1289416d8220SZaibo Xu sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1290416d8220SZaibo Xu
1291416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1292416d8220SZaibo Xu SEC_CMODE_OFFSET);
1293416d8220SZaibo Xu sec_sqe->type2.c_alg = c_ctx->c_alg;
1294416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1295416d8220SZaibo Xu SEC_CKEY_OFFSET);
1296416d8220SZaibo Xu
1297416d8220SZaibo Xu bd_type = SEC_BD_TYPE2;
1298416d8220SZaibo Xu if (c_req->encrypt)
1299416d8220SZaibo Xu cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;
1300416d8220SZaibo Xu else
1301416d8220SZaibo Xu cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;
1302416d8220SZaibo Xu sec_sqe->type_cipher_auth = bd_type | cipher;
1303416d8220SZaibo Xu
1304adc3f65aSKai Ye /* Set destination and source address type */
1305adc3f65aSKai Ye if (req->use_pbuf) {
130674b58db8SLongfang Liu sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET;
1307adc3f65aSKai Ye da_type = SEC_PBUF << SEC_DST_SGL_OFFSET;
1308adc3f65aSKai Ye } else {
1309416d8220SZaibo Xu sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;
1310adc3f65aSKai Ye da_type = SEC_SGL << SEC_DST_SGL_OFFSET;
1311adc3f65aSKai Ye }
1312adc3f65aSKai Ye
1313adc3f65aSKai Ye sec_sqe->sdm_addr_type |= da_type;
1314416d8220SZaibo Xu scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
13159039878aSKai Ye if (req->in_dma != c_req->c_out_dma)
1316416d8220SZaibo Xu de = 0x1 << SEC_DE_OFFSET;
1317416d8220SZaibo Xu
1318416d8220SZaibo Xu sec_sqe->sds_sa_type = (de | scene | sa_type);
1319416d8220SZaibo Xu
1320416d8220SZaibo Xu sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1321416d8220SZaibo Xu sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
1322416d8220SZaibo Xu
1323416d8220SZaibo Xu return 0;
1324416d8220SZaibo Xu }
1325416d8220SZaibo Xu
sec_skcipher_bd_fill_v3(struct sec_ctx * ctx,struct sec_req * req)1326adc3f65aSKai Ye static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1327adc3f65aSKai Ye {
1328adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1329adc3f65aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1330adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req;
1331adc3f65aSKai Ye u32 bd_param = 0;
1332adc3f65aSKai Ye u16 cipher;
1333adc3f65aSKai Ye
1334adc3f65aSKai Ye memset(sec_sqe3, 0, sizeof(struct sec_sqe3));
1335adc3f65aSKai Ye
1336adc3f65aSKai Ye sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1337adc3f65aSKai Ye sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
13389039878aSKai Ye sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1339adc3f65aSKai Ye sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1340adc3f65aSKai Ye
1341adc3f65aSKai Ye sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1342adc3f65aSKai Ye c_ctx->c_mode;
1343adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1344adc3f65aSKai Ye SEC_CKEY_OFFSET_V3);
1345adc3f65aSKai Ye
1346adc3f65aSKai Ye if (c_req->encrypt)
1347adc3f65aSKai Ye cipher = SEC_CIPHER_ENC;
1348adc3f65aSKai Ye else
1349adc3f65aSKai Ye cipher = SEC_CIPHER_DEC;
1350adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(cipher);
1351adc3f65aSKai Ye
1352566f060fSKai Ye /* Set the CTR counter mode is 128bit rollover */
1353566f060fSKai Ye sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER <<
1354566f060fSKai Ye SEC_CTR_CNT_OFFSET);
1355566f060fSKai Ye
1356adc3f65aSKai Ye if (req->use_pbuf) {
1357adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3;
1358adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3;
1359adc3f65aSKai Ye } else {
1360adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3;
1361adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3;
1362adc3f65aSKai Ye }
1363adc3f65aSKai Ye
1364adc3f65aSKai Ye bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
13659039878aSKai Ye if (req->in_dma != c_req->c_out_dma)
1366adc3f65aSKai Ye bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1367adc3f65aSKai Ye
1368adc3f65aSKai Ye bd_param |= SEC_BD_TYPE3;
1369adc3f65aSKai Ye sec_sqe3->bd_param = cpu_to_le32(bd_param);
1370adc3f65aSKai Ye
1371adc3f65aSKai Ye sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len);
1372dd1a502cSQi Tao sec_sqe3->tag = cpu_to_le64((unsigned long)req);
1373adc3f65aSKai Ye
1374adc3f65aSKai Ye return 0;
1375adc3f65aSKai Ye }
1376adc3f65aSKai Ye
13777b44c0eeSKai Ye /* increment counter (128-bit int) */
ctr_iv_inc(__u8 * counter,__u8 bits,__u32 nums)13787b44c0eeSKai Ye static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
13797b44c0eeSKai Ye {
13807b44c0eeSKai Ye do {
13817b44c0eeSKai Ye --bits;
13827b44c0eeSKai Ye nums += counter[bits];
13837b44c0eeSKai Ye counter[bits] = nums & BITS_MASK;
13847b44c0eeSKai Ye nums >>= BYTE_BITS;
13857b44c0eeSKai Ye } while (bits && nums);
13867b44c0eeSKai Ye }
13877b44c0eeSKai Ye
sec_update_iv(struct sec_req * req,enum sec_alg_type alg_type)13882f072d75SZaibo Xu static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
1389416d8220SZaibo Xu {
13902f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req;
1391416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req;
1392416d8220SZaibo Xu u32 iv_size = req->ctx->c_ctx.ivsize;
1393416d8220SZaibo Xu struct scatterlist *sgl;
13942f072d75SZaibo Xu unsigned int cryptlen;
1395416d8220SZaibo Xu size_t sz;
13962f072d75SZaibo Xu u8 *iv;
1397416d8220SZaibo Xu
1398416d8220SZaibo Xu if (req->c_req.encrypt)
13992f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
1400416d8220SZaibo Xu else
14012f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
1402416d8220SZaibo Xu
14032f072d75SZaibo Xu if (alg_type == SEC_SKCIPHER) {
14042f072d75SZaibo Xu iv = sk_req->iv;
14052f072d75SZaibo Xu cryptlen = sk_req->cryptlen;
14062f072d75SZaibo Xu } else {
14072f072d75SZaibo Xu iv = aead_req->iv;
14082f072d75SZaibo Xu cryptlen = aead_req->cryptlen;
14092f072d75SZaibo Xu }
14102f072d75SZaibo Xu
14117b44c0eeSKai Ye if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) {
14122f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
14132f072d75SZaibo Xu cryptlen - iv_size);
1414b9c8d897SZaibo Xu if (unlikely(sz != iv_size))
1415a44dce50SLongfang Liu dev_err(req->ctx->dev, "copy output iv error!\n");
14167b44c0eeSKai Ye } else {
14177b44c0eeSKai Ye sz = cryptlen / iv_size;
14187b44c0eeSKai Ye if (cryptlen % iv_size)
14197b44c0eeSKai Ye sz += 1;
14207b44c0eeSKai Ye ctr_iv_inc(iv, iv_size, sz);
14217b44c0eeSKai Ye }
1422416d8220SZaibo Xu }
1423416d8220SZaibo Xu
sec_back_req_clear(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)14249597efc3SKai Ye static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
14259597efc3SKai Ye struct sec_qp_ctx *qp_ctx)
14269597efc3SKai Ye {
14279597efc3SKai Ye struct sec_req *backlog_req = NULL;
14289597efc3SKai Ye
142902884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
14309597efc3SKai Ye if (ctx->fake_req_limit >=
14319597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) &&
14329597efc3SKai Ye !list_empty(&qp_ctx->backlog)) {
14339597efc3SKai Ye backlog_req = list_first_entry(&qp_ctx->backlog,
14349597efc3SKai Ye typeof(*backlog_req), backlog_head);
14359597efc3SKai Ye list_del(&backlog_req->backlog_head);
14369597efc3SKai Ye }
143702884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
14389597efc3SKai Ye
14399597efc3SKai Ye return backlog_req;
14409597efc3SKai Ye }
14419597efc3SKai Ye
sec_skcipher_callback(struct sec_ctx * ctx,struct sec_req * req,int err)1442310ea0acSZaibo Xu static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1443310ea0acSZaibo Xu int err)
1444416d8220SZaibo Xu {
1445416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req;
1446416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
14479597efc3SKai Ye struct skcipher_request *backlog_sk_req;
14489597efc3SKai Ye struct sec_req *backlog_req;
1449416d8220SZaibo Xu
1450416d8220SZaibo Xu sec_free_req_id(req);
1451416d8220SZaibo Xu
14527b44c0eeSKai Ye /* IV output at encrypto of CBC/CTR mode */
14537b44c0eeSKai Ye if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
14547b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
14552f072d75SZaibo Xu sec_update_iv(req, SEC_SKCIPHER);
1456416d8220SZaibo Xu
14579597efc3SKai Ye while (1) {
14589597efc3SKai Ye backlog_req = sec_back_req_clear(ctx, qp_ctx);
14599597efc3SKai Ye if (!backlog_req)
14609597efc3SKai Ye break;
14619597efc3SKai Ye
14629597efc3SKai Ye backlog_sk_req = backlog_req->c_req.sk_req;
1463e2b537b7SHerbert Xu skcipher_request_complete(backlog_sk_req, -EINPROGRESS);
14649597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);
14659597efc3SKai Ye }
14669597efc3SKai Ye
1467e2b537b7SHerbert Xu skcipher_request_complete(sk_req, err);
1468416d8220SZaibo Xu }
1469416d8220SZaibo Xu
set_aead_auth_iv(struct sec_ctx * ctx,struct sec_req * req)1470c16a70c1SKai Ye static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
14712f072d75SZaibo Xu {
14722f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req;
14732514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1474c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
1475c16a70c1SKai Ye size_t authsize = ctx->a_ctx.mac_len;
1476c16a70c1SKai Ye u32 data_size = aead_req->cryptlen;
1477c16a70c1SKai Ye u8 flage = 0;
1478c16a70c1SKai Ye u8 cm, cl;
1479c16a70c1SKai Ye
1480c16a70c1SKai Ye /* the specification has been checked in aead_iv_demension_check() */
1481c16a70c1SKai Ye cl = c_req->c_ivin[0] + 1;
1482c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00;
1483c16a70c1SKai Ye memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl);
1484c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT;
1485c16a70c1SKai Ye
1486c16a70c1SKai Ye /* the last 3bit is L' */
1487c16a70c1SKai Ye flage |= c_req->c_ivin[0] & IV_CL_MASK;
1488c16a70c1SKai Ye
1489c16a70c1SKai Ye /* the M' is bit3~bit5, the Flags is bit6 */
1490c16a70c1SKai Ye cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM;
1491c16a70c1SKai Ye flage |= cm << IV_CM_OFFSET;
1492c16a70c1SKai Ye if (aead_req->assoclen)
1493c16a70c1SKai Ye flage |= 0x01 << IV_FLAGS_OFFSET;
1494c16a70c1SKai Ye
1495c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize);
1496c16a70c1SKai Ye a_req->a_ivin[0] = flage;
1497c16a70c1SKai Ye
1498c16a70c1SKai Ye /*
1499c16a70c1SKai Ye * the last 32bit is counter's initial number,
1500c16a70c1SKai Ye * but the nonce uses the first 16bit
1501c16a70c1SKai Ye * the tail 16bit fill with the cipher length
1502c16a70c1SKai Ye */
1503c16a70c1SKai Ye if (!c_req->encrypt)
1504c16a70c1SKai Ye data_size = aead_req->cryptlen - authsize;
1505c16a70c1SKai Ye
1506c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] =
1507c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK;
1508c16a70c1SKai Ye data_size >>= IV_BYTE_OFFSET;
1509c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] =
1510c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK;
1511c16a70c1SKai Ye }
1512c16a70c1SKai Ye
sec_aead_set_iv(struct sec_ctx * ctx,struct sec_req * req)1513c16a70c1SKai Ye static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req)
1514c16a70c1SKai Ye {
1515c16a70c1SKai Ye struct aead_request *aead_req = req->aead_req.aead_req;
1516c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
1517c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm);
1518c16a70c1SKai Ye struct sec_cipher_req *c_req = &req->c_req;
1519c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
15202f072d75SZaibo Xu
15212514f559SLongfang Liu memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize);
1522c16a70c1SKai Ye
1523c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) {
1524c16a70c1SKai Ye /*
1525c16a70c1SKai Ye * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter},
1526c16a70c1SKai Ye * the counter must set to 0x01
1527c16a70c1SKai Ye */
1528c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize;
1529c16a70c1SKai Ye /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */
1530c16a70c1SKai Ye set_aead_auth_iv(ctx, req);
1531c16a70c1SKai Ye }
1532c16a70c1SKai Ye
1533c16a70c1SKai Ye /* GCM 12Byte Cipher_IV == Auth_IV */
1534c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) {
1535c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize;
1536c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE);
1537c16a70c1SKai Ye }
1538c16a70c1SKai Ye }
1539c16a70c1SKai Ye
sec_auth_bd_fill_xcm(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe * sec_sqe)1540c16a70c1SKai Ye static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir,
1541c16a70c1SKai Ye struct sec_req *req, struct sec_sqe *sec_sqe)
1542c16a70c1SKai Ye {
1543c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
1544c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req;
1545c16a70c1SKai Ye
1546c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1547c16a70c1SKai Ye sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len);
1548c16a70c1SKai Ye
1549c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1550c16a70c1SKai Ye sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr;
1551c16a70c1SKai Ye sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1552c16a70c1SKai Ye sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET;
1553c16a70c1SKai Ye
1554c16a70c1SKai Ye if (dir)
1555c16a70c1SKai Ye sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1556c16a70c1SKai Ye else
1557c16a70c1SKai Ye sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1558c16a70c1SKai Ye
1559c16a70c1SKai Ye sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen);
1560c16a70c1SKai Ye sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0);
1561c16a70c1SKai Ye sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1562c16a70c1SKai Ye
1563c16a70c1SKai Ye sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1564c16a70c1SKai Ye }
1565c16a70c1SKai Ye
sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe3 * sqe3)1566c16a70c1SKai Ye static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir,
1567c16a70c1SKai Ye struct sec_req *req, struct sec_sqe3 *sqe3)
1568c16a70c1SKai Ye {
1569c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
1570c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req;
1571c16a70c1SKai Ye
1572c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1573c16a70c1SKai Ye sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3);
1574c16a70c1SKai Ye
1575c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1576c16a70c1SKai Ye sqe3->a_key_addr = sqe3->c_key_addr;
1577c16a70c1SKai Ye sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1578c16a70c1SKai Ye sqe3->auth_mac_key |= SEC_NO_AUTH;
1579c16a70c1SKai Ye
1580c16a70c1SKai Ye if (dir)
1581c16a70c1SKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1582c16a70c1SKai Ye else
1583c16a70c1SKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1584c16a70c1SKai Ye
1585c16a70c1SKai Ye sqe3->a_len_key = cpu_to_le32(aq->assoclen);
1586c16a70c1SKai Ye sqe3->auth_src_offset = cpu_to_le16(0x0);
1587c16a70c1SKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1588c16a70c1SKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
15892f072d75SZaibo Xu }
15902f072d75SZaibo Xu
sec_auth_bd_fill_ex(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe * sec_sqe)15912f072d75SZaibo Xu static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,
15922f072d75SZaibo Xu struct sec_req *req, struct sec_sqe *sec_sqe)
15932f072d75SZaibo Xu {
15942f072d75SZaibo Xu struct sec_aead_req *a_req = &req->aead_req;
15952f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
15962f072d75SZaibo Xu struct aead_request *aq = a_req->aead_req;
15972f072d75SZaibo Xu
15982f072d75SZaibo Xu sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);
15992f072d75SZaibo Xu
16002f072d75SZaibo Xu sec_sqe->type2.mac_key_alg =
16012f072d75SZaibo Xu cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE);
16022f072d75SZaibo Xu
16032f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |=
16042f072d75SZaibo Xu cpu_to_le32((u32)((ctx->a_key_len) /
16052f072d75SZaibo Xu SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET);
16062f072d75SZaibo Xu
16072f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |=
16082f072d75SZaibo Xu cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);
16092f072d75SZaibo Xu
1610668f1ab7SKai Ye if (dir) {
16112f072d75SZaibo Xu sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;
16122f072d75SZaibo Xu sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1613668f1ab7SKai Ye } else {
1614668f1ab7SKai Ye sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET;
16152f072d75SZaibo Xu sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1616668f1ab7SKai Ye }
16172f072d75SZaibo Xu sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);
16182f072d75SZaibo Xu
16192f072d75SZaibo Xu sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
16202f072d75SZaibo Xu
16212514f559SLongfang Liu sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
16222f072d75SZaibo Xu }
16232f072d75SZaibo Xu
sec_aead_bd_fill(struct sec_ctx * ctx,struct sec_req * req)16242f072d75SZaibo Xu static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
16252f072d75SZaibo Xu {
16262f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
16272f072d75SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe;
16282f072d75SZaibo Xu int ret;
16292f072d75SZaibo Xu
16302f072d75SZaibo Xu ret = sec_skcipher_bd_fill(ctx, req);
16312f072d75SZaibo Xu if (unlikely(ret)) {
1632a44dce50SLongfang Liu dev_err(ctx->dev, "skcipher bd fill is error!\n");
16332f072d75SZaibo Xu return ret;
16342f072d75SZaibo Xu }
16352f072d75SZaibo Xu
1636c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1637c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1638c16a70c1SKai Ye sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1639c16a70c1SKai Ye else
16402f072d75SZaibo Xu sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);
16412f072d75SZaibo Xu
16422f072d75SZaibo Xu return 0;
16432f072d75SZaibo Xu }
16442f072d75SZaibo Xu
sec_auth_bd_fill_ex_v3(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe3 * sqe3)1645adc3f65aSKai Ye static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir,
1646adc3f65aSKai Ye struct sec_req *req, struct sec_sqe3 *sqe3)
1647adc3f65aSKai Ye {
1648adc3f65aSKai Ye struct sec_aead_req *a_req = &req->aead_req;
1649adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req;
1650adc3f65aSKai Ye struct aead_request *aq = a_req->aead_req;
1651adc3f65aSKai Ye
1652adc3f65aSKai Ye sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma);
1653adc3f65aSKai Ye
1654adc3f65aSKai Ye sqe3->auth_mac_key |=
1655adc3f65aSKai Ye cpu_to_le32((u32)(ctx->mac_len /
1656adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3);
1657adc3f65aSKai Ye
1658adc3f65aSKai Ye sqe3->auth_mac_key |=
1659adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_key_len /
1660adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3);
1661adc3f65aSKai Ye
1662adc3f65aSKai Ye sqe3->auth_mac_key |=
1663adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3);
1664adc3f65aSKai Ye
1665adc3f65aSKai Ye if (dir) {
1666adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);
1667adc3f65aSKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1668adc3f65aSKai Ye } else {
166960ef3ddeSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2);
1670adc3f65aSKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1671adc3f65aSKai Ye }
1672adc3f65aSKai Ye sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen);
1673adc3f65aSKai Ye
1674adc3f65aSKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1675adc3f65aSKai Ye
1676adc3f65aSKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1677adc3f65aSKai Ye }
1678adc3f65aSKai Ye
sec_aead_bd_fill_v3(struct sec_ctx * ctx,struct sec_req * req)1679adc3f65aSKai Ye static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1680adc3f65aSKai Ye {
1681adc3f65aSKai Ye struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1682adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1683adc3f65aSKai Ye int ret;
1684adc3f65aSKai Ye
1685adc3f65aSKai Ye ret = sec_skcipher_bd_fill_v3(ctx, req);
1686adc3f65aSKai Ye if (unlikely(ret)) {
1687adc3f65aSKai Ye dev_err(ctx->dev, "skcipher bd3 fill is error!\n");
1688adc3f65aSKai Ye return ret;
1689adc3f65aSKai Ye }
1690adc3f65aSKai Ye
1691c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1692c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1693c16a70c1SKai Ye sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt,
1694c16a70c1SKai Ye req, sec_sqe3);
1695c16a70c1SKai Ye else
1696c16a70c1SKai Ye sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt,
1697c16a70c1SKai Ye req, sec_sqe3);
1698adc3f65aSKai Ye
1699adc3f65aSKai Ye return 0;
1700adc3f65aSKai Ye }
1701adc3f65aSKai Ye
sec_aead_callback(struct sec_ctx * c,struct sec_req * req,int err)17022f072d75SZaibo Xu static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
17032f072d75SZaibo Xu {
17042f072d75SZaibo Xu struct aead_request *a_req = req->aead_req.aead_req;
17052f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
17062514f559SLongfang Liu struct sec_aead_req *aead_req = &req->aead_req;
17072f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
17082f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm);
17092f072d75SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
17109597efc3SKai Ye struct aead_request *backlog_aead_req;
17119597efc3SKai Ye struct sec_req *backlog_req;
17122f072d75SZaibo Xu size_t sz;
17132f072d75SZaibo Xu
17142f072d75SZaibo Xu if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
17152f072d75SZaibo Xu sec_update_iv(req, SEC_AEAD);
17162f072d75SZaibo Xu
17172f072d75SZaibo Xu /* Copy output mac */
17182f072d75SZaibo Xu if (!err && c_req->encrypt) {
17192f072d75SZaibo Xu struct scatterlist *sgl = a_req->dst;
17202f072d75SZaibo Xu
17212f072d75SZaibo Xu sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl),
17222514f559SLongfang Liu aead_req->out_mac,
17232f072d75SZaibo Xu authsize, a_req->cryptlen +
17242f072d75SZaibo Xu a_req->assoclen);
17252f072d75SZaibo Xu if (unlikely(sz != authsize)) {
1726a44dce50SLongfang Liu dev_err(c->dev, "copy out mac err!\n");
17272f072d75SZaibo Xu err = -EINVAL;
17282f072d75SZaibo Xu }
17292f072d75SZaibo Xu }
17302f072d75SZaibo Xu
17312f072d75SZaibo Xu sec_free_req_id(req);
17322f072d75SZaibo Xu
17339597efc3SKai Ye while (1) {
17349597efc3SKai Ye backlog_req = sec_back_req_clear(c, qp_ctx);
17359597efc3SKai Ye if (!backlog_req)
17369597efc3SKai Ye break;
17379597efc3SKai Ye
17389597efc3SKai Ye backlog_aead_req = backlog_req->aead_req.aead_req;
1739e2b537b7SHerbert Xu aead_request_complete(backlog_aead_req, -EINPROGRESS);
17409597efc3SKai Ye atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);
17419597efc3SKai Ye }
17422f072d75SZaibo Xu
1743e2b537b7SHerbert Xu aead_request_complete(a_req, err);
17442f072d75SZaibo Xu }
17452f072d75SZaibo Xu
sec_request_uninit(struct sec_ctx * ctx,struct sec_req * req)1746416d8220SZaibo Xu static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1747416d8220SZaibo Xu {
1748416d8220SZaibo Xu sec_free_req_id(req);
1749a181647cSZaibo Xu sec_free_queue_id(ctx, req);
1750416d8220SZaibo Xu }
1751416d8220SZaibo Xu
sec_request_init(struct sec_ctx * ctx,struct sec_req * req)1752416d8220SZaibo Xu static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1753416d8220SZaibo Xu {
1754416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx;
17557c7d902aSZaibo Xu int queue_id;
1756416d8220SZaibo Xu
1757416d8220SZaibo Xu /* To load balance */
1758a181647cSZaibo Xu queue_id = sec_alloc_queue_id(ctx, req);
1759a181647cSZaibo Xu qp_ctx = &ctx->qp_ctx[queue_id];
1760416d8220SZaibo Xu
1761416d8220SZaibo Xu req->req_id = sec_alloc_req_id(req, qp_ctx);
1762b9c8d897SZaibo Xu if (unlikely(req->req_id < 0)) {
1763a181647cSZaibo Xu sec_free_queue_id(ctx, req);
1764416d8220SZaibo Xu return req->req_id;
1765416d8220SZaibo Xu }
1766416d8220SZaibo Xu
17677c7d902aSZaibo Xu return 0;
1768416d8220SZaibo Xu }
1769416d8220SZaibo Xu
sec_process(struct sec_ctx * ctx,struct sec_req * req)1770416d8220SZaibo Xu static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1771416d8220SZaibo Xu {
17722514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1773416d8220SZaibo Xu int ret;
1774416d8220SZaibo Xu
1775416d8220SZaibo Xu ret = sec_request_init(ctx, req);
1776b9c8d897SZaibo Xu if (unlikely(ret))
1777416d8220SZaibo Xu return ret;
1778416d8220SZaibo Xu
1779416d8220SZaibo Xu ret = sec_request_transfer(ctx, req);
1780b9c8d897SZaibo Xu if (unlikely(ret))
1781416d8220SZaibo Xu goto err_uninit_req;
1782416d8220SZaibo Xu
1783416d8220SZaibo Xu /* Output IV as decrypto */
17847b44c0eeSKai Ye if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
17857b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR))
17862f072d75SZaibo Xu sec_update_iv(req, ctx->alg_type);
1787416d8220SZaibo Xu
1788416d8220SZaibo Xu ret = ctx->req_op->bd_send(ctx, req);
17899597efc3SKai Ye if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
17909597efc3SKai Ye (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1791a44dce50SLongfang Liu dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1792416d8220SZaibo Xu goto err_send_req;
1793416d8220SZaibo Xu }
1794416d8220SZaibo Xu
1795416d8220SZaibo Xu return ret;
1796416d8220SZaibo Xu
1797416d8220SZaibo Xu err_send_req:
1798416d8220SZaibo Xu /* As failing, restore the IV from user */
17992f072d75SZaibo Xu if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
18002f072d75SZaibo Xu if (ctx->alg_type == SEC_SKCIPHER)
18012514f559SLongfang Liu memcpy(req->c_req.sk_req->iv, c_req->c_ivin,
1802416d8220SZaibo Xu ctx->c_ctx.ivsize);
18032f072d75SZaibo Xu else
18042514f559SLongfang Liu memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,
18052f072d75SZaibo Xu ctx->c_ctx.ivsize);
18062f072d75SZaibo Xu }
1807416d8220SZaibo Xu
1808416d8220SZaibo Xu sec_request_untransfer(ctx, req);
1809416d8220SZaibo Xu err_uninit_req:
1810416d8220SZaibo Xu sec_request_uninit(ctx, req);
1811416d8220SZaibo Xu return ret;
1812416d8220SZaibo Xu }
1813416d8220SZaibo Xu
1814a181647cSZaibo Xu static const struct sec_req_op sec_skcipher_req_ops = {
1815416d8220SZaibo Xu .buf_map = sec_skcipher_sgl_map,
1816416d8220SZaibo Xu .buf_unmap = sec_skcipher_sgl_unmap,
1817416d8220SZaibo Xu .do_transfer = sec_skcipher_copy_iv,
1818416d8220SZaibo Xu .bd_fill = sec_skcipher_bd_fill,
1819416d8220SZaibo Xu .bd_send = sec_bd_send,
1820416d8220SZaibo Xu .callback = sec_skcipher_callback,
1821416d8220SZaibo Xu .process = sec_process,
1822416d8220SZaibo Xu };
1823416d8220SZaibo Xu
18242f072d75SZaibo Xu static const struct sec_req_op sec_aead_req_ops = {
18252f072d75SZaibo Xu .buf_map = sec_aead_sgl_map,
18262f072d75SZaibo Xu .buf_unmap = sec_aead_sgl_unmap,
1827c16a70c1SKai Ye .do_transfer = sec_aead_set_iv,
18282f072d75SZaibo Xu .bd_fill = sec_aead_bd_fill,
18292f072d75SZaibo Xu .bd_send = sec_bd_send,
18302f072d75SZaibo Xu .callback = sec_aead_callback,
18312f072d75SZaibo Xu .process = sec_process,
18322f072d75SZaibo Xu };
18332f072d75SZaibo Xu
1834adc3f65aSKai Ye static const struct sec_req_op sec_skcipher_req_ops_v3 = {
1835adc3f65aSKai Ye .buf_map = sec_skcipher_sgl_map,
1836adc3f65aSKai Ye .buf_unmap = sec_skcipher_sgl_unmap,
1837adc3f65aSKai Ye .do_transfer = sec_skcipher_copy_iv,
1838adc3f65aSKai Ye .bd_fill = sec_skcipher_bd_fill_v3,
1839adc3f65aSKai Ye .bd_send = sec_bd_send,
1840adc3f65aSKai Ye .callback = sec_skcipher_callback,
1841adc3f65aSKai Ye .process = sec_process,
1842adc3f65aSKai Ye };
1843adc3f65aSKai Ye
1844adc3f65aSKai Ye static const struct sec_req_op sec_aead_req_ops_v3 = {
1845adc3f65aSKai Ye .buf_map = sec_aead_sgl_map,
1846adc3f65aSKai Ye .buf_unmap = sec_aead_sgl_unmap,
1847c16a70c1SKai Ye .do_transfer = sec_aead_set_iv,
1848adc3f65aSKai Ye .bd_fill = sec_aead_bd_fill_v3,
1849adc3f65aSKai Ye .bd_send = sec_bd_send,
1850adc3f65aSKai Ye .callback = sec_aead_callback,
1851adc3f65aSKai Ye .process = sec_process,
1852adc3f65aSKai Ye };
1853adc3f65aSKai Ye
sec_skcipher_ctx_init(struct crypto_skcipher * tfm)1854416d8220SZaibo Xu static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)
1855416d8220SZaibo Xu {
1856416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
1857adc3f65aSKai Ye int ret;
1858416d8220SZaibo Xu
1859adc3f65aSKai Ye ret = sec_skcipher_init(tfm);
1860adc3f65aSKai Ye if (ret)
1861adc3f65aSKai Ye return ret;
1862adc3f65aSKai Ye
1863adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) {
1864adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2;
1865a181647cSZaibo Xu ctx->req_op = &sec_skcipher_req_ops;
1866adc3f65aSKai Ye } else {
1867adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3;
1868adc3f65aSKai Ye ctx->req_op = &sec_skcipher_req_ops_v3;
1869adc3f65aSKai Ye }
1870416d8220SZaibo Xu
1871adc3f65aSKai Ye return ret;
1872416d8220SZaibo Xu }
1873416d8220SZaibo Xu
sec_skcipher_ctx_exit(struct crypto_skcipher * tfm)1874416d8220SZaibo Xu static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)
1875416d8220SZaibo Xu {
1876a181647cSZaibo Xu sec_skcipher_uninit(tfm);
1877416d8220SZaibo Xu }
1878416d8220SZaibo Xu
sec_aead_init(struct crypto_aead * tfm)18792f072d75SZaibo Xu static int sec_aead_init(struct crypto_aead *tfm)
18802f072d75SZaibo Xu {
18812f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
18822f072d75SZaibo Xu int ret;
18832f072d75SZaibo Xu
18842f072d75SZaibo Xu crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
18852f072d75SZaibo Xu ctx->alg_type = SEC_AEAD;
18862f072d75SZaibo Xu ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
1887c16a70c1SKai Ye if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
1888c16a70c1SKai Ye ctx->c_ctx.ivsize > SEC_IV_SIZE) {
1889c16a70c1SKai Ye pr_err("get error aead iv size!\n");
18902f072d75SZaibo Xu return -EINVAL;
18912f072d75SZaibo Xu }
18922f072d75SZaibo Xu
18932f072d75SZaibo Xu ret = sec_ctx_base_init(ctx);
18942f072d75SZaibo Xu if (ret)
18952f072d75SZaibo Xu return ret;
1896adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) {
1897adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2;
1898adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops;
1899adc3f65aSKai Ye } else {
1900adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3;
1901adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops_v3;
1902adc3f65aSKai Ye }
19032f072d75SZaibo Xu
19042f072d75SZaibo Xu ret = sec_auth_init(ctx);
19052f072d75SZaibo Xu if (ret)
19062f072d75SZaibo Xu goto err_auth_init;
19072f072d75SZaibo Xu
19082f072d75SZaibo Xu ret = sec_cipher_init(ctx);
19092f072d75SZaibo Xu if (ret)
19102f072d75SZaibo Xu goto err_cipher_init;
19112f072d75SZaibo Xu
19122f072d75SZaibo Xu return ret;
19132f072d75SZaibo Xu
19142f072d75SZaibo Xu err_cipher_init:
19152f072d75SZaibo Xu sec_auth_uninit(ctx);
19162f072d75SZaibo Xu err_auth_init:
19172f072d75SZaibo Xu sec_ctx_base_uninit(ctx);
19182f072d75SZaibo Xu return ret;
19192f072d75SZaibo Xu }
19202f072d75SZaibo Xu
sec_aead_exit(struct crypto_aead * tfm)19212f072d75SZaibo Xu static void sec_aead_exit(struct crypto_aead *tfm)
19222f072d75SZaibo Xu {
19232f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19242f072d75SZaibo Xu
19252f072d75SZaibo Xu sec_cipher_uninit(ctx);
19262f072d75SZaibo Xu sec_auth_uninit(ctx);
19272f072d75SZaibo Xu sec_ctx_base_uninit(ctx);
19282f072d75SZaibo Xu }
19292f072d75SZaibo Xu
sec_aead_ctx_init(struct crypto_aead * tfm,const char * hash_name)19302f072d75SZaibo Xu static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
19312f072d75SZaibo Xu {
19322f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19332f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
19342f072d75SZaibo Xu int ret;
19352f072d75SZaibo Xu
19362f072d75SZaibo Xu ret = sec_aead_init(tfm);
19372f072d75SZaibo Xu if (ret) {
19382f072d75SZaibo Xu pr_err("hisi_sec2: aead init error!\n");
19392f072d75SZaibo Xu return ret;
19402f072d75SZaibo Xu }
19412f072d75SZaibo Xu
19422f072d75SZaibo Xu auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
19432f072d75SZaibo Xu if (IS_ERR(auth_ctx->hash_tfm)) {
1944a44dce50SLongfang Liu dev_err(ctx->dev, "aead alloc shash error!\n");
19452f072d75SZaibo Xu sec_aead_exit(tfm);
19462f072d75SZaibo Xu return PTR_ERR(auth_ctx->hash_tfm);
19472f072d75SZaibo Xu }
19482f072d75SZaibo Xu
19492f072d75SZaibo Xu return 0;
19502f072d75SZaibo Xu }
19512f072d75SZaibo Xu
sec_aead_ctx_exit(struct crypto_aead * tfm)19522f072d75SZaibo Xu static void sec_aead_ctx_exit(struct crypto_aead *tfm)
19532f072d75SZaibo Xu {
19542f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19552f072d75SZaibo Xu
19562f072d75SZaibo Xu crypto_free_shash(ctx->a_ctx.hash_tfm);
19572f072d75SZaibo Xu sec_aead_exit(tfm);
19582f072d75SZaibo Xu }
19592f072d75SZaibo Xu
sec_aead_xcm_ctx_init(struct crypto_aead * tfm)1960c16a70c1SKai Ye static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm)
1961c16a70c1SKai Ye {
19626c46a329SKai Ye struct aead_alg *alg = crypto_aead_alg(tfm);
1963c16a70c1SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19646c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
19656c46a329SKai Ye const char *aead_name = alg->base.cra_name;
1966c16a70c1SKai Ye int ret;
1967c16a70c1SKai Ye
1968c16a70c1SKai Ye ret = sec_aead_init(tfm);
1969c16a70c1SKai Ye if (ret) {
1970c16a70c1SKai Ye dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n");
1971c16a70c1SKai Ye return ret;
1972c16a70c1SKai Ye }
1973c16a70c1SKai Ye
19746c46a329SKai Ye a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,
19756c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK |
19766c46a329SKai Ye CRYPTO_ALG_ASYNC);
19776c46a329SKai Ye if (IS_ERR(a_ctx->fallback_aead_tfm)) {
19786c46a329SKai Ye dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");
19796c46a329SKai Ye sec_aead_exit(tfm);
19806c46a329SKai Ye return PTR_ERR(a_ctx->fallback_aead_tfm);
19816c46a329SKai Ye }
19826c46a329SKai Ye a_ctx->fallback = false;
19836c46a329SKai Ye
1984c16a70c1SKai Ye return 0;
1985c16a70c1SKai Ye }
1986c16a70c1SKai Ye
sec_aead_xcm_ctx_exit(struct crypto_aead * tfm)1987c16a70c1SKai Ye static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm)
1988c16a70c1SKai Ye {
19896c46a329SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19906c46a329SKai Ye
19916c46a329SKai Ye crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);
1992c16a70c1SKai Ye sec_aead_exit(tfm);
1993c16a70c1SKai Ye }
1994c16a70c1SKai Ye
sec_aead_sha1_ctx_init(struct crypto_aead * tfm)19952f072d75SZaibo Xu static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)
19962f072d75SZaibo Xu {
19972f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha1");
19982f072d75SZaibo Xu }
19992f072d75SZaibo Xu
sec_aead_sha256_ctx_init(struct crypto_aead * tfm)20002f072d75SZaibo Xu static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)
20012f072d75SZaibo Xu {
20022f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha256");
20032f072d75SZaibo Xu }
20042f072d75SZaibo Xu
sec_aead_sha512_ctx_init(struct crypto_aead * tfm)20052f072d75SZaibo Xu static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
20062f072d75SZaibo Xu {
20072f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha512");
20082f072d75SZaibo Xu }
20092f072d75SZaibo Xu
sec_skcipher_cryptlen_check(struct sec_ctx * ctx,struct sec_req * sreq)20102132d4efSKai Ye static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx,
2011059c5342SLongfang Liu struct sec_req *sreq)
2012059c5342SLongfang Liu {
2013059c5342SLongfang Liu u32 cryptlen = sreq->c_req.sk_req->cryptlen;
2014059c5342SLongfang Liu struct device *dev = ctx->dev;
2015059c5342SLongfang Liu u8 c_mode = ctx->c_ctx.c_mode;
2016059c5342SLongfang Liu int ret = 0;
2017059c5342SLongfang Liu
2018059c5342SLongfang Liu switch (c_mode) {
2019059c5342SLongfang Liu case SEC_CMODE_XTS:
2020059c5342SLongfang Liu if (unlikely(cryptlen < AES_BLOCK_SIZE)) {
2021059c5342SLongfang Liu dev_err(dev, "skcipher XTS mode input length error!\n");
2022059c5342SLongfang Liu ret = -EINVAL;
2023059c5342SLongfang Liu }
2024059c5342SLongfang Liu break;
2025059c5342SLongfang Liu case SEC_CMODE_ECB:
2026059c5342SLongfang Liu case SEC_CMODE_CBC:
2027059c5342SLongfang Liu if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {
2028059c5342SLongfang Liu dev_err(dev, "skcipher AES input length error!\n");
2029059c5342SLongfang Liu ret = -EINVAL;
2030059c5342SLongfang Liu }
2031059c5342SLongfang Liu break;
20327b44c0eeSKai Ye case SEC_CMODE_CTR:
20337b44c0eeSKai Ye if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) {
20347b44c0eeSKai Ye dev_err(dev, "skcipher HW version error!\n");
20357b44c0eeSKai Ye ret = -EINVAL;
20367b44c0eeSKai Ye }
20377b44c0eeSKai Ye break;
2038059c5342SLongfang Liu default:
2039059c5342SLongfang Liu ret = -EINVAL;
2040059c5342SLongfang Liu }
2041059c5342SLongfang Liu
2042059c5342SLongfang Liu return ret;
2043059c5342SLongfang Liu }
2044059c5342SLongfang Liu
sec_skcipher_param_check(struct sec_ctx * ctx,struct sec_req * sreq)2045d6de2a59SZaibo Xu static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
2046416d8220SZaibo Xu {
2047d6de2a59SZaibo Xu struct skcipher_request *sk_req = sreq->c_req.sk_req;
2048a44dce50SLongfang Liu struct device *dev = ctx->dev;
2049d6de2a59SZaibo Xu u8 c_alg = ctx->c_ctx.c_alg;
2050416d8220SZaibo Xu
20517b44c0eeSKai Ye if (unlikely(!sk_req->src || !sk_req->dst ||
20527b44c0eeSKai Ye sk_req->cryptlen > MAX_INPUT_DATA_LEN)) {
2053416d8220SZaibo Xu dev_err(dev, "skcipher input param error!\n");
2054416d8220SZaibo Xu return -EINVAL;
2055416d8220SZaibo Xu }
2056d6de2a59SZaibo Xu sreq->c_req.c_len = sk_req->cryptlen;
205774b58db8SLongfang Liu
205874b58db8SLongfang Liu if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ)
205974b58db8SLongfang Liu sreq->use_pbuf = true;
206074b58db8SLongfang Liu else
206174b58db8SLongfang Liu sreq->use_pbuf = false;
206274b58db8SLongfang Liu
2063416d8220SZaibo Xu if (c_alg == SEC_CALG_3DES) {
2064b9c8d897SZaibo Xu if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {
2065416d8220SZaibo Xu dev_err(dev, "skcipher 3des input length error!\n");
2066416d8220SZaibo Xu return -EINVAL;
2067416d8220SZaibo Xu }
2068416d8220SZaibo Xu return 0;
2069416d8220SZaibo Xu } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
20702132d4efSKai Ye return sec_skcipher_cryptlen_check(ctx, sreq);
2071416d8220SZaibo Xu }
2072059c5342SLongfang Liu
2073416d8220SZaibo Xu dev_err(dev, "skcipher algorithm error!\n");
2074633e507fSLongfang Liu
2075416d8220SZaibo Xu return -EINVAL;
2076416d8220SZaibo Xu }
2077416d8220SZaibo Xu
sec_skcipher_soft_crypto(struct sec_ctx * ctx,struct skcipher_request * sreq,bool encrypt)20785652d55aSKai Ye static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
20795652d55aSKai Ye struct skcipher_request *sreq, bool encrypt)
20805652d55aSKai Ye {
20815652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
2082e764d81dSKai Ye SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm);
20835652d55aSKai Ye struct device *dev = ctx->dev;
20845652d55aSKai Ye int ret;
20855652d55aSKai Ye
20865652d55aSKai Ye if (!c_ctx->fbtfm) {
2087e764d81dSKai Ye dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n");
20885652d55aSKai Ye return -EINVAL;
20895652d55aSKai Ye }
20905652d55aSKai Ye
20915652d55aSKai Ye skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm);
20925652d55aSKai Ye
20935652d55aSKai Ye /* software need sync mode to do crypto */
20945652d55aSKai Ye skcipher_request_set_callback(subreq, sreq->base.flags,
20955652d55aSKai Ye NULL, NULL);
20965652d55aSKai Ye skcipher_request_set_crypt(subreq, sreq->src, sreq->dst,
20975652d55aSKai Ye sreq->cryptlen, sreq->iv);
20985652d55aSKai Ye if (encrypt)
20995652d55aSKai Ye ret = crypto_skcipher_encrypt(subreq);
21005652d55aSKai Ye else
21015652d55aSKai Ye ret = crypto_skcipher_decrypt(subreq);
21025652d55aSKai Ye
21035652d55aSKai Ye skcipher_request_zero(subreq);
21045652d55aSKai Ye
21055652d55aSKai Ye return ret;
21065652d55aSKai Ye }
21075652d55aSKai Ye
sec_skcipher_crypto(struct skcipher_request * sk_req,bool encrypt)2108416d8220SZaibo Xu static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2109416d8220SZaibo Xu {
2110416d8220SZaibo Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2111416d8220SZaibo Xu struct sec_req *req = skcipher_request_ctx(sk_req);
2112416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2113416d8220SZaibo Xu int ret;
2114416d8220SZaibo Xu
21155652d55aSKai Ye if (!sk_req->cryptlen) {
21165652d55aSKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_XTS)
21175652d55aSKai Ye return -EINVAL;
2118416d8220SZaibo Xu return 0;
21195652d55aSKai Ye }
2120416d8220SZaibo Xu
21219597efc3SKai Ye req->flag = sk_req->base.flags;
2122416d8220SZaibo Xu req->c_req.sk_req = sk_req;
2123416d8220SZaibo Xu req->c_req.encrypt = encrypt;
2124416d8220SZaibo Xu req->ctx = ctx;
2125416d8220SZaibo Xu
2126d6de2a59SZaibo Xu ret = sec_skcipher_param_check(ctx, req);
2127d6de2a59SZaibo Xu if (unlikely(ret))
2128d6de2a59SZaibo Xu return -EINVAL;
2129d6de2a59SZaibo Xu
21305652d55aSKai Ye if (unlikely(ctx->c_ctx.fallback))
21315652d55aSKai Ye return sec_skcipher_soft_crypto(ctx, sk_req, encrypt);
21325652d55aSKai Ye
2133416d8220SZaibo Xu return ctx->req_op->process(ctx, req);
2134416d8220SZaibo Xu }
2135416d8220SZaibo Xu
sec_skcipher_encrypt(struct skcipher_request * sk_req)2136416d8220SZaibo Xu static int sec_skcipher_encrypt(struct skcipher_request *sk_req)
2137416d8220SZaibo Xu {
2138416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, true);
2139416d8220SZaibo Xu }
2140416d8220SZaibo Xu
sec_skcipher_decrypt(struct skcipher_request * sk_req)2141416d8220SZaibo Xu static int sec_skcipher_decrypt(struct skcipher_request *sk_req)
2142416d8220SZaibo Xu {
2143416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, false);
2144416d8220SZaibo Xu }
2145416d8220SZaibo Xu
2146c4af4225SQi Tao #define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \
2147c4af4225SQi Tao sec_min_key_size, sec_max_key_size, blk_size, iv_size)\
2148416d8220SZaibo Xu {\
2149416d8220SZaibo Xu .base = {\
2150416d8220SZaibo Xu .cra_name = sec_cra_name,\
2151416d8220SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\
2152416d8220SZaibo Xu .cra_priority = SEC_PRIORITY,\
21535652d55aSKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\
21545652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK,\
2155416d8220SZaibo Xu .cra_blocksize = blk_size,\
2156416d8220SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\
2157416d8220SZaibo Xu .cra_module = THIS_MODULE,\
2158416d8220SZaibo Xu },\
2159c4af4225SQi Tao .init = sec_skcipher_ctx_init,\
2160c4af4225SQi Tao .exit = sec_skcipher_ctx_exit,\
2161416d8220SZaibo Xu .setkey = sec_set_key,\
2162416d8220SZaibo Xu .decrypt = sec_skcipher_decrypt,\
2163416d8220SZaibo Xu .encrypt = sec_skcipher_encrypt,\
2164416d8220SZaibo Xu .min_keysize = sec_min_key_size,\
2165416d8220SZaibo Xu .max_keysize = sec_max_key_size,\
2166416d8220SZaibo Xu .ivsize = iv_size,\
2167921715b6SWenkai Lin }
2168416d8220SZaibo Xu
2169921715b6SWenkai Lin static struct sec_skcipher sec_skciphers[] = {
2170921715b6SWenkai Lin {
2171921715b6SWenkai Lin .alg_msk = BIT(0),
2172921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE,
2173921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0),
2174921715b6SWenkai Lin },
2175921715b6SWenkai Lin {
2176921715b6SWenkai Lin .alg_msk = BIT(1),
2177921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE,
2178921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2179921715b6SWenkai Lin },
2180921715b6SWenkai Lin {
2181921715b6SWenkai Lin .alg_msk = BIT(2),
2182921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE,
2183921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2184921715b6SWenkai Lin },
2185921715b6SWenkai Lin {
2186921715b6SWenkai Lin .alg_msk = BIT(3),
2187921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE,
2188921715b6SWenkai Lin SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2189921715b6SWenkai Lin },
2190921715b6SWenkai Lin {
2191921715b6SWenkai Lin .alg_msk = BIT(12),
2192921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE,
2193921715b6SWenkai Lin AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2194921715b6SWenkai Lin },
2195921715b6SWenkai Lin {
2196921715b6SWenkai Lin .alg_msk = BIT(13),
2197921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE,
2198921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2199921715b6SWenkai Lin },
2200921715b6SWenkai Lin {
2201921715b6SWenkai Lin .alg_msk = BIT(14),
2202921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE,
2203921715b6SWenkai Lin SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2204921715b6SWenkai Lin },
2205921715b6SWenkai Lin {
2206921715b6SWenkai Lin .alg_msk = BIT(23),
2207921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE,
2208921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0),
2209921715b6SWenkai Lin },
2210921715b6SWenkai Lin {
2211921715b6SWenkai Lin .alg_msk = BIT(24),
2212921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE,
2213921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE,
2214921715b6SWenkai Lin DES3_EDE_BLOCK_SIZE),
2215921715b6SWenkai Lin },
22167b44c0eeSKai Ye };
22177b44c0eeSKai Ye
aead_iv_demension_check(struct aead_request * aead_req)2218c16a70c1SKai Ye static int aead_iv_demension_check(struct aead_request *aead_req)
2219c16a70c1SKai Ye {
2220c16a70c1SKai Ye u8 cl;
2221c16a70c1SKai Ye
2222c16a70c1SKai Ye cl = aead_req->iv[0] + 1;
2223c16a70c1SKai Ye if (cl < IV_CL_MIN || cl > IV_CL_MAX)
2224c16a70c1SKai Ye return -EINVAL;
2225c16a70c1SKai Ye
2226c16a70c1SKai Ye if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl))
2227c16a70c1SKai Ye return -EOVERFLOW;
2228c16a70c1SKai Ye
2229c16a70c1SKai Ye return 0;
2230c16a70c1SKai Ye }
2231c16a70c1SKai Ye
sec_aead_spec_check(struct sec_ctx * ctx,struct sec_req * sreq)2232c16a70c1SKai Ye static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq)
2233c16a70c1SKai Ye {
2234c16a70c1SKai Ye struct aead_request *req = sreq->aead_req.aead_req;
2235c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2236c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm);
2237c16a70c1SKai Ye u8 c_mode = ctx->c_ctx.c_mode;
2238c16a70c1SKai Ye struct device *dev = ctx->dev;
2239c16a70c1SKai Ye int ret;
2240c16a70c1SKai Ye
2241c16a70c1SKai Ye if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN ||
2242c16a70c1SKai Ye req->assoclen > SEC_MAX_AAD_LEN)) {
2243c16a70c1SKai Ye dev_err(dev, "aead input spec error!\n");
2244c16a70c1SKai Ye return -EINVAL;
2245c16a70c1SKai Ye }
2246c16a70c1SKai Ye
2247c16a70c1SKai Ye if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) ||
2248c16a70c1SKai Ye (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN ||
2249c16a70c1SKai Ye authsize & MAC_LEN_MASK)))) {
2250c16a70c1SKai Ye dev_err(dev, "aead input mac length error!\n");
2251c16a70c1SKai Ye return -EINVAL;
2252c16a70c1SKai Ye }
2253c16a70c1SKai Ye
2254c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM) {
22555e340558SKai Ye if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) {
22565e340558SKai Ye dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n");
22575e340558SKai Ye return -EINVAL;
22585e340558SKai Ye }
2259c16a70c1SKai Ye ret = aead_iv_demension_check(req);
2260c16a70c1SKai Ye if (ret) {
2261c16a70c1SKai Ye dev_err(dev, "aead input iv param error!\n");
2262c16a70c1SKai Ye return ret;
2263c16a70c1SKai Ye }
2264c16a70c1SKai Ye }
2265c16a70c1SKai Ye
2266c16a70c1SKai Ye if (sreq->c_req.encrypt)
2267c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen;
2268c16a70c1SKai Ye else
2269c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen - authsize;
2270c16a70c1SKai Ye if (c_mode == SEC_CMODE_CBC) {
2271c16a70c1SKai Ye if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
2272c16a70c1SKai Ye dev_err(dev, "aead crypto length error!\n");
2273c16a70c1SKai Ye return -EINVAL;
2274c16a70c1SKai Ye }
2275c16a70c1SKai Ye }
2276c16a70c1SKai Ye
2277c16a70c1SKai Ye return 0;
2278c16a70c1SKai Ye }
2279c16a70c1SKai Ye
sec_aead_param_check(struct sec_ctx * ctx,struct sec_req * sreq)22802f072d75SZaibo Xu static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
22812f072d75SZaibo Xu {
22822f072d75SZaibo Xu struct aead_request *req = sreq->aead_req.aead_req;
22832f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(req);
22842f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm);
2285a44dce50SLongfang Liu struct device *dev = ctx->dev;
2286a44dce50SLongfang Liu u8 c_alg = ctx->c_ctx.c_alg;
22872f072d75SZaibo Xu
2288c16a70c1SKai Ye if (unlikely(!req->src || !req->dst)) {
2289a44dce50SLongfang Liu dev_err(dev, "aead input param error!\n");
22902f072d75SZaibo Xu return -EINVAL;
22912f072d75SZaibo Xu }
22922f072d75SZaibo Xu
2293c16a70c1SKai Ye if (ctx->sec->qm.ver == QM_HW_V2) {
2294c16a70c1SKai Ye if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt &&
2295c16a70c1SKai Ye req->cryptlen <= authsize))) {
22966c46a329SKai Ye ctx->a_ctx.fallback = true;
2297c16a70c1SKai Ye return -EINVAL;
2298c16a70c1SKai Ye }
2299c16a70c1SKai Ye }
2300c16a70c1SKai Ye
2301c16a70c1SKai Ye /* Support AES or SM4 */
2302c16a70c1SKai Ye if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) {
2303c16a70c1SKai Ye dev_err(dev, "aead crypto alg error!\n");
2304c16a70c1SKai Ye return -EINVAL;
2305c16a70c1SKai Ye }
2306c16a70c1SKai Ye
2307c16a70c1SKai Ye if (unlikely(sec_aead_spec_check(ctx, sreq)))
2308c16a70c1SKai Ye return -EINVAL;
2309c16a70c1SKai Ye
231074b58db8SLongfang Liu if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <=
231174b58db8SLongfang Liu SEC_PBUF_SZ)
231274b58db8SLongfang Liu sreq->use_pbuf = true;
231374b58db8SLongfang Liu else
231474b58db8SLongfang Liu sreq->use_pbuf = false;
231574b58db8SLongfang Liu
23162f072d75SZaibo Xu return 0;
23172f072d75SZaibo Xu }
23182f072d75SZaibo Xu
sec_aead_soft_crypto(struct sec_ctx * ctx,struct aead_request * aead_req,bool encrypt)23196c46a329SKai Ye static int sec_aead_soft_crypto(struct sec_ctx *ctx,
23206c46a329SKai Ye struct aead_request *aead_req,
23216c46a329SKai Ye bool encrypt)
23226c46a329SKai Ye {
23236c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
23246c46a329SKai Ye struct device *dev = ctx->dev;
23250a2a464fSKai Ye struct aead_request *subreq;
23260a2a464fSKai Ye int ret;
23276c46a329SKai Ye
23286c46a329SKai Ye /* Kunpeng920 aead mode not support input 0 size */
23296c46a329SKai Ye if (!a_ctx->fallback_aead_tfm) {
233084c2c729SColin Ian King dev_err(dev, "aead fallback tfm is NULL!\n");
23316c46a329SKai Ye return -EINVAL;
23326c46a329SKai Ye }
23336c46a329SKai Ye
23340a2a464fSKai Ye subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL);
23350a2a464fSKai Ye if (!subreq)
23360a2a464fSKai Ye return -ENOMEM;
23370a2a464fSKai Ye
23386c46a329SKai Ye aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);
23396c46a329SKai Ye aead_request_set_callback(subreq, aead_req->base.flags,
23406c46a329SKai Ye aead_req->base.complete, aead_req->base.data);
23416c46a329SKai Ye aead_request_set_crypt(subreq, aead_req->src, aead_req->dst,
23426c46a329SKai Ye aead_req->cryptlen, aead_req->iv);
23436c46a329SKai Ye aead_request_set_ad(subreq, aead_req->assoclen);
23446c46a329SKai Ye
23450a2a464fSKai Ye if (encrypt)
23460a2a464fSKai Ye ret = crypto_aead_encrypt(subreq);
23470a2a464fSKai Ye else
23480a2a464fSKai Ye ret = crypto_aead_decrypt(subreq);
23490a2a464fSKai Ye aead_request_free(subreq);
23500a2a464fSKai Ye
23510a2a464fSKai Ye return ret;
23526c46a329SKai Ye }
23536c46a329SKai Ye
sec_aead_crypto(struct aead_request * a_req,bool encrypt)23542f072d75SZaibo Xu static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
23552f072d75SZaibo Xu {
23562f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
23572f072d75SZaibo Xu struct sec_req *req = aead_request_ctx(a_req);
23582f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
23592f072d75SZaibo Xu int ret;
23602f072d75SZaibo Xu
23619597efc3SKai Ye req->flag = a_req->base.flags;
23622f072d75SZaibo Xu req->aead_req.aead_req = a_req;
23632f072d75SZaibo Xu req->c_req.encrypt = encrypt;
23642f072d75SZaibo Xu req->ctx = ctx;
23652f072d75SZaibo Xu
23662f072d75SZaibo Xu ret = sec_aead_param_check(ctx, req);
23676c46a329SKai Ye if (unlikely(ret)) {
23686c46a329SKai Ye if (ctx->a_ctx.fallback)
23696c46a329SKai Ye return sec_aead_soft_crypto(ctx, a_req, encrypt);
23702f072d75SZaibo Xu return -EINVAL;
23716c46a329SKai Ye }
23722f072d75SZaibo Xu
23732f072d75SZaibo Xu return ctx->req_op->process(ctx, req);
23742f072d75SZaibo Xu }
23752f072d75SZaibo Xu
sec_aead_encrypt(struct aead_request * a_req)23762f072d75SZaibo Xu static int sec_aead_encrypt(struct aead_request *a_req)
23772f072d75SZaibo Xu {
23782f072d75SZaibo Xu return sec_aead_crypto(a_req, true);
23792f072d75SZaibo Xu }
23802f072d75SZaibo Xu
sec_aead_decrypt(struct aead_request * a_req)23812f072d75SZaibo Xu static int sec_aead_decrypt(struct aead_request *a_req)
23822f072d75SZaibo Xu {
23832f072d75SZaibo Xu return sec_aead_crypto(a_req, false);
23842f072d75SZaibo Xu }
23852f072d75SZaibo Xu
2386c16a70c1SKai Ye #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\
23872f072d75SZaibo Xu ctx_exit, blk_size, iv_size, max_authsize)\
23882f072d75SZaibo Xu {\
23892f072d75SZaibo Xu .base = {\
23902f072d75SZaibo Xu .cra_name = sec_cra_name,\
23912f072d75SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\
23922f072d75SZaibo Xu .cra_priority = SEC_PRIORITY,\
23936c46a329SKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\
23946c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK,\
23952f072d75SZaibo Xu .cra_blocksize = blk_size,\
23962f072d75SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\
23972f072d75SZaibo Xu .cra_module = THIS_MODULE,\
23982f072d75SZaibo Xu },\
23992f072d75SZaibo Xu .init = ctx_init,\
24002f072d75SZaibo Xu .exit = ctx_exit,\
24012f072d75SZaibo Xu .setkey = sec_set_key,\
24026c46a329SKai Ye .setauthsize = sec_aead_setauthsize,\
24032f072d75SZaibo Xu .decrypt = sec_aead_decrypt,\
24042f072d75SZaibo Xu .encrypt = sec_aead_encrypt,\
24052f072d75SZaibo Xu .ivsize = iv_size,\
24062f072d75SZaibo Xu .maxauthsize = max_authsize,\
24072f072d75SZaibo Xu }
24082f072d75SZaibo Xu
2409921715b6SWenkai Lin static struct sec_aead sec_aeads[] = {
2410921715b6SWenkai Lin {
2411921715b6SWenkai Lin .alg_msk = BIT(6),
2412921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init,
2413921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2414921715b6SWenkai Lin AES_BLOCK_SIZE),
2415921715b6SWenkai Lin },
2416921715b6SWenkai Lin {
2417921715b6SWenkai Lin .alg_msk = BIT(7),
2418921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init,
2419921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2420921715b6SWenkai Lin AES_BLOCK_SIZE),
2421921715b6SWenkai Lin },
2422921715b6SWenkai Lin {
2423921715b6SWenkai Lin .alg_msk = BIT(17),
2424921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init,
2425921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2426921715b6SWenkai Lin AES_BLOCK_SIZE),
2427921715b6SWenkai Lin },
2428921715b6SWenkai Lin {
2429921715b6SWenkai Lin .alg_msk = BIT(18),
2430921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init,
2431921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2432921715b6SWenkai Lin AES_BLOCK_SIZE),
2433921715b6SWenkai Lin },
2434921715b6SWenkai Lin {
2435921715b6SWenkai Lin .alg_msk = BIT(43),
2436921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1,
2437921715b6SWenkai Lin sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2438c16a70c1SKai Ye AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),
2439921715b6SWenkai Lin },
2440921715b6SWenkai Lin {
2441921715b6SWenkai Lin .alg_msk = BIT(44),
2442921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256,
2443921715b6SWenkai Lin sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2444c16a70c1SKai Ye AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),
2445921715b6SWenkai Lin },
2446921715b6SWenkai Lin {
2447921715b6SWenkai Lin .alg_msk = BIT(45),
2448921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512,
2449921715b6SWenkai Lin sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2450c16a70c1SKai Ye AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
2451921715b6SWenkai Lin },
2452c16a70c1SKai Ye };
2453c16a70c1SKai Ye
sec_unregister_skcipher(u64 alg_mask,int end)2454921715b6SWenkai Lin static void sec_unregister_skcipher(u64 alg_mask, int end)
2455921715b6SWenkai Lin {
2456921715b6SWenkai Lin int i;
2457c16a70c1SKai Ye
2458921715b6SWenkai Lin for (i = 0; i < end; i++)
2459921715b6SWenkai Lin if (sec_skciphers[i].alg_msk & alg_mask)
2460921715b6SWenkai Lin crypto_unregister_skcipher(&sec_skciphers[i].alg);
2461921715b6SWenkai Lin }
2462921715b6SWenkai Lin
sec_register_skcipher(u64 alg_mask)2463921715b6SWenkai Lin static int sec_register_skcipher(u64 alg_mask)
2464921715b6SWenkai Lin {
2465921715b6SWenkai Lin int i, ret, count;
2466921715b6SWenkai Lin
2467921715b6SWenkai Lin count = ARRAY_SIZE(sec_skciphers);
2468921715b6SWenkai Lin
2469921715b6SWenkai Lin for (i = 0; i < count; i++) {
2470921715b6SWenkai Lin if (!(sec_skciphers[i].alg_msk & alg_mask))
2471921715b6SWenkai Lin continue;
2472921715b6SWenkai Lin
2473921715b6SWenkai Lin ret = crypto_register_skcipher(&sec_skciphers[i].alg);
2474921715b6SWenkai Lin if (ret)
2475921715b6SWenkai Lin goto err;
2476921715b6SWenkai Lin }
2477921715b6SWenkai Lin
2478921715b6SWenkai Lin return 0;
2479921715b6SWenkai Lin
2480921715b6SWenkai Lin err:
2481921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, i);
2482921715b6SWenkai Lin
2483921715b6SWenkai Lin return ret;
2484921715b6SWenkai Lin }
2485921715b6SWenkai Lin
sec_unregister_aead(u64 alg_mask,int end)2486921715b6SWenkai Lin static void sec_unregister_aead(u64 alg_mask, int end)
2487921715b6SWenkai Lin {
2488921715b6SWenkai Lin int i;
2489921715b6SWenkai Lin
2490921715b6SWenkai Lin for (i = 0; i < end; i++)
2491921715b6SWenkai Lin if (sec_aeads[i].alg_msk & alg_mask)
2492921715b6SWenkai Lin crypto_unregister_aead(&sec_aeads[i].alg);
2493921715b6SWenkai Lin }
2494921715b6SWenkai Lin
sec_register_aead(u64 alg_mask)2495921715b6SWenkai Lin static int sec_register_aead(u64 alg_mask)
2496921715b6SWenkai Lin {
2497921715b6SWenkai Lin int i, ret, count;
2498921715b6SWenkai Lin
2499921715b6SWenkai Lin count = ARRAY_SIZE(sec_aeads);
2500921715b6SWenkai Lin
2501921715b6SWenkai Lin for (i = 0; i < count; i++) {
2502921715b6SWenkai Lin if (!(sec_aeads[i].alg_msk & alg_mask))
2503921715b6SWenkai Lin continue;
2504921715b6SWenkai Lin
2505921715b6SWenkai Lin ret = crypto_register_aead(&sec_aeads[i].alg);
2506921715b6SWenkai Lin if (ret)
2507921715b6SWenkai Lin goto err;
2508921715b6SWenkai Lin }
2509921715b6SWenkai Lin
2510921715b6SWenkai Lin return 0;
2511921715b6SWenkai Lin
2512921715b6SWenkai Lin err:
2513921715b6SWenkai Lin sec_unregister_aead(alg_mask, i);
2514921715b6SWenkai Lin
2515921715b6SWenkai Lin return ret;
2516921715b6SWenkai Lin }
25172f072d75SZaibo Xu
sec_register_to_crypto(struct hisi_qm * qm)25188123455aSMeng Yu int sec_register_to_crypto(struct hisi_qm *qm)
2519416d8220SZaibo Xu {
2520f1115b00SZhiqi Song u64 alg_mask;
2521b42ab1c6SWeili Qian int ret = 0;
2522b42ab1c6SWeili Qian
2523f1115b00SZhiqi Song alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,
2524f1115b00SZhiqi Song SEC_DRV_ALG_BITMAP_LOW_IDX);
2525f1115b00SZhiqi Song
2526b42ab1c6SWeili Qian mutex_lock(&sec_algs_lock);
2527b42ab1c6SWeili Qian if (sec_available_devs) {
2528b42ab1c6SWeili Qian sec_available_devs++;
2529b42ab1c6SWeili Qian goto unlock;
2530b42ab1c6SWeili Qian }
2531416d8220SZaibo Xu
2532921715b6SWenkai Lin ret = sec_register_skcipher(alg_mask);
25332f072d75SZaibo Xu if (ret)
2534b42ab1c6SWeili Qian goto unlock;
25352f072d75SZaibo Xu
2536921715b6SWenkai Lin ret = sec_register_aead(alg_mask);
25377b44c0eeSKai Ye if (ret)
2538b42ab1c6SWeili Qian goto unreg_skcipher;
2539c16a70c1SKai Ye
2540b42ab1c6SWeili Qian sec_available_devs++;
2541b42ab1c6SWeili Qian mutex_unlock(&sec_algs_lock);
2542b42ab1c6SWeili Qian
2543b42ab1c6SWeili Qian return 0;
2544b42ab1c6SWeili Qian
2545b42ab1c6SWeili Qian unreg_skcipher:
2546b42ab1c6SWeili Qian sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2547b42ab1c6SWeili Qian unlock:
2548b42ab1c6SWeili Qian mutex_unlock(&sec_algs_lock);
2549416d8220SZaibo Xu return ret;
2550416d8220SZaibo Xu }
2551416d8220SZaibo Xu
sec_unregister_from_crypto(struct hisi_qm * qm)25528123455aSMeng Yu void sec_unregister_from_crypto(struct hisi_qm *qm)
2553416d8220SZaibo Xu {
2554f1115b00SZhiqi Song u64 alg_mask;
2555f1115b00SZhiqi Song
2556f1115b00SZhiqi Song alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,
2557f1115b00SZhiqi Song SEC_DRV_ALG_BITMAP_LOW_IDX);
2558c16a70c1SKai Ye
2559b42ab1c6SWeili Qian mutex_lock(&sec_algs_lock);
2560b42ab1c6SWeili Qian if (--sec_available_devs)
2561b42ab1c6SWeili Qian goto unlock;
2562b42ab1c6SWeili Qian
2563921715b6SWenkai Lin sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads));
2564921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2565b42ab1c6SWeili Qian
2566b42ab1c6SWeili Qian unlock:
2567b42ab1c6SWeili Qian mutex_unlock(&sec_algs_lock);
25682f072d75SZaibo Xu }
2569