1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3
4 #include <crypto/aes.h>
5 #include <crypto/aead.h>
6 #include <crypto/algapi.h>
7 #include <crypto/authenc.h>
8 #include <crypto/des.h>
9 #include <crypto/hash.h>
10 #include <crypto/internal/aead.h>
11 #include <crypto/internal/des.h>
12 #include <crypto/sha1.h>
13 #include <crypto/sha2.h>
14 #include <crypto/skcipher.h>
15 #include <crypto/xts.h>
16 #include <linux/crypto.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/idr.h>
19
20 #include "sec.h"
21 #include "sec_crypto.h"
22
23 #define SEC_PRIORITY 4001
24 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE)
25 #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE)
26 #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE)
27 #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE)
28 #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE)
29
30 /* SEC sqe(bd) bit operational relative MACRO */
31 #define SEC_DE_OFFSET 1
32 #define SEC_CIPHER_OFFSET 4
33 #define SEC_SCENE_OFFSET 3
34 #define SEC_DST_SGL_OFFSET 2
35 #define SEC_SRC_SGL_OFFSET 7
36 #define SEC_CKEY_OFFSET 9
37 #define SEC_CMODE_OFFSET 12
38 #define SEC_AKEY_OFFSET 5
39 #define SEC_AEAD_ALG_OFFSET 11
40 #define SEC_AUTH_OFFSET 6
41
42 #define SEC_DE_OFFSET_V3 9
43 #define SEC_SCENE_OFFSET_V3 5
44 #define SEC_CKEY_OFFSET_V3 13
45 #define SEC_CTR_CNT_OFFSET 25
46 #define SEC_CTR_CNT_ROLLOVER 2
47 #define SEC_SRC_SGL_OFFSET_V3 11
48 #define SEC_DST_SGL_OFFSET_V3 14
49 #define SEC_CALG_OFFSET_V3 4
50 #define SEC_AKEY_OFFSET_V3 9
51 #define SEC_MAC_OFFSET_V3 4
52 #define SEC_AUTH_ALG_OFFSET_V3 15
53 #define SEC_CIPHER_AUTH_V3 0xbf
54 #define SEC_AUTH_CIPHER_V3 0x40
55 #define SEC_FLAG_OFFSET 7
56 #define SEC_FLAG_MASK 0x0780
57 #define SEC_DONE_MASK 0x0001
58 #define SEC_ICV_MASK 0x000E
59
60 #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth))
61 #define SEC_SGL_SGE_NR 128
62 #define SEC_CIPHER_AUTH 0xfe
63 #define SEC_AUTH_CIPHER 0x1
64 #define SEC_MAX_MAC_LEN 64
65 #define SEC_MAX_AAD_LEN 65535
66 #define SEC_MAX_CCM_AAD_LEN 65279
67 #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))
68
69 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ
70 #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE)
71 #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \
72 SEC_MAX_MAC_LEN * 2)
73 #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG)
74 #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM)
75 #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \
76 SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM))
77 #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \
78 SEC_PBUF_LEFT_SZ(depth))
79
80 #define SEC_SQE_CFLAG 2
81 #define SEC_SQE_AEAD_FLAG 3
82 #define SEC_SQE_DONE 0x1
83 #define SEC_ICV_ERR 0x2
84 #define MAC_LEN_MASK 0x1U
85 #define MAX_INPUT_DATA_LEN 0xFFFE00
86 #define BITS_MASK 0xFF
87 #define WORD_MASK 0x3
88 #define BYTE_BITS 0x8
89 #define BYTES_TO_WORDS(bcount) ((bcount) >> 2)
90 #define SEC_XTS_NAME_SZ 0x3
91 #define IV_CM_CAL_NUM 2
92 #define IV_CL_MASK 0x7
93 #define IV_CL_MIN 2
94 #define IV_CL_MID 4
95 #define IV_CL_MAX 8
96 #define IV_FLAGS_OFFSET 0x6
97 #define IV_CM_OFFSET 0x3
98 #define IV_LAST_BYTE1 1
99 #define IV_LAST_BYTE2 2
100 #define IV_LAST_BYTE_MASK 0xFF
101 #define IV_CTR_INIT 0x1
102 #define IV_BYTE_OFFSET 0x8
103 #define SEC_GCM_MIN_AUTH_SZ 0x8
104 #define SEC_RETRY_MAX_CNT 5U
105
106 static DEFINE_MUTEX(sec_algs_lock);
107 static unsigned int sec_available_devs;
108
109 struct sec_skcipher {
110 u64 alg_msk;
111 struct skcipher_alg alg;
112 };
113
114 struct sec_aead {
115 u64 alg_msk;
116 struct aead_alg alg;
117 };
118
119 static int sec_aead_soft_crypto(struct sec_ctx *ctx,
120 struct aead_request *aead_req,
121 bool encrypt);
122 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
123 struct skcipher_request *sreq, bool encrypt);
124
sec_alloc_req_id(struct sec_req * req,struct sec_qp_ctx * qp_ctx)125 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
126 {
127 int req_id;
128
129 spin_lock_bh(&qp_ctx->id_lock);
130 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);
131 spin_unlock_bh(&qp_ctx->id_lock);
132 return req_id;
133 }
134
sec_free_req_id(struct sec_req * req)135 static void sec_free_req_id(struct sec_req *req)
136 {
137 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
138 int req_id = req->req_id;
139
140 if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) {
141 dev_err(req->ctx->dev, "free request id invalid!\n");
142 return;
143 }
144
145 spin_lock_bh(&qp_ctx->id_lock);
146 idr_remove(&qp_ctx->req_idr, req_id);
147 spin_unlock_bh(&qp_ctx->id_lock);
148 }
149
pre_parse_finished_bd(struct bd_status * status,void * resp)150 static void pre_parse_finished_bd(struct bd_status *status, void *resp)
151 {
152 struct sec_sqe *bd = resp;
153
154 status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
155 status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1;
156 status->flag = (le16_to_cpu(bd->type2.done_flag) &
157 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
158 status->tag = le16_to_cpu(bd->type2.tag);
159 status->err_type = bd->type2.error_type;
160 }
161
pre_parse_finished_bd3(struct bd_status * status,void * resp)162 static void pre_parse_finished_bd3(struct bd_status *status, void *resp)
163 {
164 struct sec_sqe3 *bd3 = resp;
165
166 status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK;
167 status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1;
168 status->flag = (le16_to_cpu(bd3->done_flag) &
169 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
170 status->tag = le64_to_cpu(bd3->tag);
171 status->err_type = bd3->error_type;
172 }
173
sec_cb_status_check(struct sec_req * req,struct bd_status * status)174 static int sec_cb_status_check(struct sec_req *req,
175 struct bd_status *status)
176 {
177 struct sec_ctx *ctx = req->ctx;
178
179 if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) {
180 dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n",
181 req->err_type, status->done);
182 return -EIO;
183 }
184
185 if (unlikely(ctx->alg_type == SEC_SKCIPHER)) {
186 if (unlikely(status->flag != SEC_SQE_CFLAG)) {
187 dev_err_ratelimited(ctx->dev, "flag[%u]\n",
188 status->flag);
189 return -EIO;
190 }
191 } else if (unlikely(ctx->alg_type == SEC_AEAD)) {
192 if (unlikely(status->flag != SEC_SQE_AEAD_FLAG ||
193 status->icv == SEC_ICV_ERR)) {
194 dev_err_ratelimited(ctx->dev,
195 "flag[%u], icv[%u]\n",
196 status->flag, status->icv);
197 return -EBADMSG;
198 }
199 }
200
201 return 0;
202 }
203
qp_send_message(struct sec_req * req)204 static int qp_send_message(struct sec_req *req)
205 {
206 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
207 int ret;
208
209 if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1)
210 return -EBUSY;
211
212 spin_lock_bh(&qp_ctx->req_lock);
213 if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) {
214 spin_unlock_bh(&qp_ctx->req_lock);
215 return -EBUSY;
216 }
217
218 if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) {
219 req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head);
220 qp_ctx->req_list[qp_ctx->send_head] = req;
221 }
222
223 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
224 if (ret) {
225 spin_unlock_bh(&qp_ctx->req_lock);
226 return ret;
227 }
228 if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2)
229 qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth;
230
231 spin_unlock_bh(&qp_ctx->req_lock);
232
233 atomic64_inc(&req->ctx->sec->debug.dfx.send_cnt);
234 return -EINPROGRESS;
235 }
236
sec_alg_send_backlog_soft(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)237 static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
238 {
239 struct sec_req *req, *tmp;
240 int ret;
241
242 list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) {
243 list_del(&req->list);
244 ctx->req_op->buf_unmap(ctx, req);
245 if (req->req_id >= 0)
246 sec_free_req_id(req);
247
248 if (ctx->alg_type == SEC_AEAD)
249 ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req,
250 req->c_req.encrypt);
251 else
252 ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req,
253 req->c_req.encrypt);
254
255 /* Wake up the busy thread first, then return the errno. */
256 crypto_request_complete(req->base, -EINPROGRESS);
257 crypto_request_complete(req->base, ret);
258 }
259 }
260
sec_alg_send_backlog(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)261 static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
262 {
263 struct hisi_qp *qp = qp_ctx->qp;
264 struct sec_req *req, *tmp;
265 int ret;
266
267 spin_lock_bh(&qp->backlog.lock);
268 list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) {
269 ret = qp_send_message(req);
270 switch (ret) {
271 case -EINPROGRESS:
272 list_del(&req->list);
273 crypto_request_complete(req->base, -EINPROGRESS);
274 break;
275 case -EBUSY:
276 /* Device is busy and stop send any request. */
277 goto unlock;
278 default:
279 /* Release memory resources and send all requests through software. */
280 sec_alg_send_backlog_soft(ctx, qp_ctx);
281 goto unlock;
282 }
283 }
284
285 unlock:
286 spin_unlock_bh(&qp->backlog.lock);
287 }
288
sec_req_cb(struct hisi_qp * qp,void * resp)289 static void sec_req_cb(struct hisi_qp *qp, void *resp)
290 {
291 const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head];
292 struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe);
293 struct sec_ctx *ctx = req->ctx;
294 struct sec_dfx *dfx = &ctx->sec->debug.dfx;
295 struct bd_status status;
296 int err;
297
298 pre_parse_finished_bd(&status, resp);
299
300 req->err_type = status.err_type;
301 err = sec_cb_status_check(req, &status);
302 if (err)
303 atomic64_inc(&dfx->done_flag_cnt);
304
305 atomic64_inc(&dfx->recv_cnt);
306
307 ctx->req_op->buf_unmap(ctx, req);
308 ctx->req_op->callback(ctx, req, err);
309 }
310
sec_req_cb3(struct hisi_qp * qp,void * resp)311 static void sec_req_cb3(struct hisi_qp *qp, void *resp)
312 {
313 struct bd_status status;
314 struct sec_ctx *ctx;
315 struct sec_dfx *dfx;
316 struct sec_req *req;
317 int err;
318
319 pre_parse_finished_bd3(&status, resp);
320
321 req = (void *)(uintptr_t)status.tag;
322 req->err_type = status.err_type;
323 ctx = req->ctx;
324 dfx = &ctx->sec->debug.dfx;
325
326 err = sec_cb_status_check(req, &status);
327 if (err)
328 atomic64_inc(&dfx->done_flag_cnt);
329
330 atomic64_inc(&dfx->recv_cnt);
331
332 ctx->req_op->buf_unmap(ctx, req);
333 ctx->req_op->callback(ctx, req, err);
334 }
335
sec_alg_send_message_retry(struct sec_req * req)336 static int sec_alg_send_message_retry(struct sec_req *req)
337 {
338 int ctr = 0;
339 int ret;
340
341 do {
342 ret = qp_send_message(req);
343 } while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT);
344
345 return ret;
346 }
347
sec_alg_try_enqueue(struct sec_req * req)348 static int sec_alg_try_enqueue(struct sec_req *req)
349 {
350 struct hisi_qp *qp = req->qp_ctx->qp;
351
352 /* Check if any request is already backlogged */
353 if (!list_empty(&qp->backlog.list))
354 return -EBUSY;
355
356 /* Try to enqueue to HW ring */
357 return qp_send_message(req);
358 }
359
360
sec_alg_send_message_maybacklog(struct sec_req * req)361 static int sec_alg_send_message_maybacklog(struct sec_req *req)
362 {
363 struct hisi_qp *qp = req->qp_ctx->qp;
364 int ret;
365
366 ret = sec_alg_try_enqueue(req);
367 if (ret != -EBUSY)
368 return ret;
369
370 spin_lock_bh(&qp->backlog.lock);
371 ret = sec_alg_try_enqueue(req);
372 if (ret == -EBUSY)
373 list_add_tail(&req->list, &qp->backlog.list);
374 spin_unlock_bh(&qp->backlog.lock);
375
376 return ret;
377 }
378
sec_bd_send(struct sec_ctx * ctx,struct sec_req * req)379 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
380 {
381 if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)
382 return sec_alg_send_message_maybacklog(req);
383
384 return sec_alg_send_message_retry(req);
385 }
386
sec_alloc_civ_resource(struct device * dev,struct sec_alg_res * res)387 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
388 {
389 u16 q_depth = res->depth;
390 int i;
391
392 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
393 &res->c_ivin_dma, GFP_KERNEL);
394 if (!res->c_ivin)
395 return -ENOMEM;
396
397 for (i = 1; i < q_depth; i++) {
398 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;
399 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;
400 }
401
402 return 0;
403 }
404
sec_free_civ_resource(struct device * dev,struct sec_alg_res * res)405 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)
406 {
407 if (res->c_ivin)
408 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
409 res->c_ivin, res->c_ivin_dma);
410 }
411
sec_alloc_aiv_resource(struct device * dev,struct sec_alg_res * res)412 static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res)
413 {
414 u16 q_depth = res->depth;
415 int i;
416
417 res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
418 &res->a_ivin_dma, GFP_KERNEL);
419 if (!res->a_ivin)
420 return -ENOMEM;
421
422 for (i = 1; i < q_depth; i++) {
423 res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE;
424 res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE;
425 }
426
427 return 0;
428 }
429
sec_free_aiv_resource(struct device * dev,struct sec_alg_res * res)430 static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res)
431 {
432 if (res->a_ivin)
433 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
434 res->a_ivin, res->a_ivin_dma);
435 }
436
sec_alloc_mac_resource(struct device * dev,struct sec_alg_res * res)437 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)
438 {
439 u16 q_depth = res->depth;
440 int i;
441
442 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1,
443 &res->out_mac_dma, GFP_KERNEL);
444 if (!res->out_mac)
445 return -ENOMEM;
446
447 for (i = 1; i < q_depth; i++) {
448 res[i].out_mac_dma = res->out_mac_dma +
449 i * (SEC_MAX_MAC_LEN << 1);
450 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);
451 }
452
453 return 0;
454 }
455
sec_free_mac_resource(struct device * dev,struct sec_alg_res * res)456 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)
457 {
458 if (res->out_mac)
459 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1,
460 res->out_mac, res->out_mac_dma);
461 }
462
sec_free_pbuf_resource(struct device * dev,struct sec_alg_res * res)463 static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res)
464 {
465 if (res->pbuf)
466 dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth),
467 res->pbuf, res->pbuf_dma);
468 }
469
470 /*
471 * To improve performance, pbuffer is used for
472 * small packets (< 512Bytes) as IOMMU translation using.
473 */
sec_alloc_pbuf_resource(struct device * dev,struct sec_alg_res * res)474 static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)
475 {
476 u16 q_depth = res->depth;
477 int size = SEC_PBUF_PAGE_NUM(q_depth);
478 int pbuf_page_offset;
479 int i, j, k;
480
481 res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth),
482 &res->pbuf_dma, GFP_KERNEL);
483 if (!res->pbuf)
484 return -ENOMEM;
485
486 /*
487 * SEC_PBUF_PKG contains data pbuf, iv and
488 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC>
489 * Every PAGE contains six SEC_PBUF_PKG
490 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG
491 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE
492 * for the SEC_TOTAL_PBUF_SZ
493 */
494 for (i = 0; i <= size; i++) {
495 pbuf_page_offset = PAGE_SIZE * i;
496 for (j = 0; j < SEC_PBUF_NUM; j++) {
497 k = i * SEC_PBUF_NUM + j;
498 if (k == q_depth)
499 break;
500 res[k].pbuf = res->pbuf +
501 j * SEC_PBUF_PKG + pbuf_page_offset;
502 res[k].pbuf_dma = res->pbuf_dma +
503 j * SEC_PBUF_PKG + pbuf_page_offset;
504 }
505 }
506
507 return 0;
508 }
509
sec_alg_resource_alloc(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)510 static int sec_alg_resource_alloc(struct sec_ctx *ctx,
511 struct sec_qp_ctx *qp_ctx)
512 {
513 struct sec_alg_res *res = qp_ctx->res;
514 struct device *dev = ctx->dev;
515 int ret;
516
517 ret = sec_alloc_civ_resource(dev, res);
518 if (ret)
519 return ret;
520
521 if (ctx->alg_type == SEC_AEAD) {
522 ret = sec_alloc_aiv_resource(dev, res);
523 if (ret)
524 goto alloc_aiv_fail;
525
526 ret = sec_alloc_mac_resource(dev, res);
527 if (ret)
528 goto alloc_mac_fail;
529 }
530 if (ctx->pbuf_supported) {
531 ret = sec_alloc_pbuf_resource(dev, res);
532 if (ret) {
533 dev_err(dev, "fail to alloc pbuf dma resource!\n");
534 goto alloc_pbuf_fail;
535 }
536 }
537
538 return 0;
539
540 alloc_pbuf_fail:
541 if (ctx->alg_type == SEC_AEAD)
542 sec_free_mac_resource(dev, qp_ctx->res);
543 alloc_mac_fail:
544 if (ctx->alg_type == SEC_AEAD)
545 sec_free_aiv_resource(dev, res);
546 alloc_aiv_fail:
547 sec_free_civ_resource(dev, res);
548 return ret;
549 }
550
sec_alg_resource_free(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)551 static void sec_alg_resource_free(struct sec_ctx *ctx,
552 struct sec_qp_ctx *qp_ctx)
553 {
554 struct device *dev = ctx->dev;
555
556 sec_free_civ_resource(dev, qp_ctx->res);
557
558 if (ctx->pbuf_supported)
559 sec_free_pbuf_resource(dev, qp_ctx->res);
560 if (ctx->alg_type == SEC_AEAD) {
561 sec_free_mac_resource(dev, qp_ctx->res);
562 sec_free_aiv_resource(dev, qp_ctx->res);
563 }
564 }
565
sec_alloc_qp_ctx_resource(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)566 static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
567 {
568 u16 q_depth = qp_ctx->qp->sq_depth;
569 struct device *dev = ctx->dev;
570 int ret = -ENOMEM;
571
572 qp_ctx->req_list = kzalloc_objs(struct sec_req *, q_depth);
573 if (!qp_ctx->req_list)
574 return ret;
575
576 qp_ctx->res = kzalloc_objs(struct sec_alg_res, q_depth);
577 if (!qp_ctx->res)
578 goto err_free_req_list;
579 qp_ctx->res->depth = q_depth;
580
581 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
582 if (IS_ERR(qp_ctx->c_in_pool)) {
583 dev_err(dev, "fail to create sgl pool for input!\n");
584 goto err_free_res;
585 }
586
587 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
588 if (IS_ERR(qp_ctx->c_out_pool)) {
589 dev_err(dev, "fail to create sgl pool for output!\n");
590 goto err_free_c_in_pool;
591 }
592
593 ret = sec_alg_resource_alloc(ctx, qp_ctx);
594 if (ret)
595 goto err_free_c_out_pool;
596
597 return 0;
598
599 err_free_c_out_pool:
600 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
601 err_free_c_in_pool:
602 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
603 err_free_res:
604 kfree(qp_ctx->res);
605 err_free_req_list:
606 kfree(qp_ctx->req_list);
607 return ret;
608 }
609
sec_free_qp_ctx_resource(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)610 static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
611 {
612 struct device *dev = ctx->dev;
613
614 sec_alg_resource_free(ctx, qp_ctx);
615 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
616 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
617 kfree(qp_ctx->res);
618 kfree(qp_ctx->req_list);
619 }
620
sec_create_qp_ctx(struct sec_ctx * ctx,int qp_ctx_id)621 static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id)
622 {
623 struct sec_qp_ctx *qp_ctx;
624 struct hisi_qp *qp;
625 int ret;
626
627 qp_ctx = &ctx->qp_ctx[qp_ctx_id];
628 qp = ctx->qps[qp_ctx_id];
629 qp_ctx->qp = qp;
630 qp_ctx->ctx = ctx;
631
632 if (ctx->type_supported == SEC_BD_TYPE3)
633 qp->req_cb = sec_req_cb3;
634 else
635 qp->req_cb = sec_req_cb;
636
637 spin_lock_init(&qp_ctx->req_lock);
638 idr_init(&qp_ctx->req_idr);
639 spin_lock_init(&qp_ctx->id_lock);
640 qp_ctx->send_head = 0;
641
642 ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx);
643 if (ret)
644 goto err_destroy_idr;
645
646 return 0;
647
648 err_destroy_idr:
649 idr_destroy(&qp_ctx->req_idr);
650 return ret;
651 }
652
sec_release_qp_ctx(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)653 static void sec_release_qp_ctx(struct sec_ctx *ctx,
654 struct sec_qp_ctx *qp_ctx)
655 {
656 sec_free_qp_ctx_resource(ctx, qp_ctx);
657 idr_destroy(&qp_ctx->req_idr);
658 }
659
sec_ctx_base_init(struct sec_ctx * ctx)660 static int sec_ctx_base_init(struct sec_ctx *ctx)
661 {
662 struct sec_dev *sec;
663 int i, ret;
664
665 ctx->qps = sec_create_qps();
666 if (!ctx->qps)
667 return -ENODEV;
668
669 sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);
670 ctx->sec = sec;
671 ctx->dev = &sec->qm.pdev->dev;
672 ctx->hlf_q_num = sec->ctx_q_num >> 1;
673
674 ctx->pbuf_supported = ctx->sec->iommu_used;
675 ctx->qp_ctx = kzalloc_objs(struct sec_qp_ctx, sec->ctx_q_num);
676 if (!ctx->qp_ctx) {
677 ret = -ENOMEM;
678 goto err_destroy_qps;
679 }
680
681 for (i = 0; i < sec->ctx_q_num; i++) {
682 ret = sec_create_qp_ctx(ctx, i);
683 if (ret)
684 goto err_sec_release_qp_ctx;
685 }
686
687 return 0;
688
689 err_sec_release_qp_ctx:
690 for (i = i - 1; i >= 0; i--)
691 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
692 kfree(ctx->qp_ctx);
693 err_destroy_qps:
694 sec_destroy_qps(ctx->qps, sec->ctx_q_num);
695 return ret;
696 }
697
sec_ctx_base_uninit(struct sec_ctx * ctx)698 static void sec_ctx_base_uninit(struct sec_ctx *ctx)
699 {
700 int i;
701
702 if (!ctx->qps)
703 return;
704
705 for (i = 0; i < ctx->sec->ctx_q_num; i++)
706 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
707
708 sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num);
709 kfree(ctx->qp_ctx);
710 }
711
sec_cipher_init(struct sec_ctx * ctx)712 static int sec_cipher_init(struct sec_ctx *ctx)
713 {
714 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
715
716 if (!ctx->qps)
717 return 0;
718
719 c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
720 &c_ctx->c_key_dma, GFP_KERNEL);
721 if (!c_ctx->c_key)
722 return -ENOMEM;
723
724 return 0;
725 }
726
sec_cipher_uninit(struct sec_ctx * ctx)727 static void sec_cipher_uninit(struct sec_ctx *ctx)
728 {
729 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
730
731 if (!ctx->qps)
732 return;
733
734 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
735 dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
736 c_ctx->c_key, c_ctx->c_key_dma);
737 }
738
sec_auth_init(struct sec_ctx * ctx)739 static int sec_auth_init(struct sec_ctx *ctx)
740 {
741 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
742
743 a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
744 &a_ctx->a_key_dma, GFP_KERNEL);
745 if (!a_ctx->a_key)
746 return -ENOMEM;
747
748 return 0;
749 }
750
sec_auth_uninit(struct sec_ctx * ctx)751 static void sec_auth_uninit(struct sec_ctx *ctx)
752 {
753 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
754
755 if (!ctx->qps)
756 return;
757
758 memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE);
759 dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
760 a_ctx->a_key, a_ctx->a_key_dma);
761 }
762
sec_skcipher_fbtfm_init(struct crypto_skcipher * tfm)763 static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm)
764 {
765 const char *alg = crypto_tfm_alg_name(&tfm->base);
766 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
767 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
768
769 c_ctx->fallback = false;
770
771 c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0,
772 CRYPTO_ALG_NEED_FALLBACK);
773 if (IS_ERR(c_ctx->fbtfm)) {
774 pr_err("failed to alloc fallback tfm for %s!\n", alg);
775 return PTR_ERR(c_ctx->fbtfm);
776 }
777
778 return 0;
779 }
780
sec_skcipher_init(struct crypto_skcipher * tfm)781 static int sec_skcipher_init(struct crypto_skcipher *tfm)
782 {
783 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
784 int ret;
785
786 ctx->alg_type = SEC_SKCIPHER;
787 crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req));
788 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
789 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
790 pr_err("get error skcipher iv size!\n");
791 return -EINVAL;
792 }
793
794 ret = sec_ctx_base_init(ctx);
795 if (ret && ret != -ENODEV)
796 return ret;
797
798 ret = sec_cipher_init(ctx);
799 if (ret)
800 goto err_cipher_init;
801
802 ret = sec_skcipher_fbtfm_init(tfm);
803 if (ret)
804 goto err_fbtfm_init;
805
806 return 0;
807
808 err_fbtfm_init:
809 sec_cipher_uninit(ctx);
810 err_cipher_init:
811 sec_ctx_base_uninit(ctx);
812 return ret;
813 }
814
sec_skcipher_uninit(struct crypto_skcipher * tfm)815 static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
816 {
817 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
818
819 if (ctx->c_ctx.fbtfm)
820 crypto_free_sync_skcipher(ctx->c_ctx.fbtfm);
821
822 sec_cipher_uninit(ctx);
823 sec_ctx_base_uninit(ctx);
824 }
825
sec_skcipher_3des_setkey(struct crypto_skcipher * tfm,const u8 * key,const u32 keylen)826 static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen)
827 {
828 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
829 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
830 int ret;
831
832 ret = verify_skcipher_des3_key(tfm, key);
833 if (ret)
834 return ret;
835
836 switch (keylen) {
837 case SEC_DES3_2KEY_SIZE:
838 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
839 break;
840 case SEC_DES3_3KEY_SIZE:
841 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;
842 break;
843 default:
844 return -EINVAL;
845 }
846
847 return 0;
848 }
849
sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx * c_ctx,const u32 keylen,const enum sec_cmode c_mode)850 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,
851 const u32 keylen,
852 const enum sec_cmode c_mode)
853 {
854 if (c_mode == SEC_CMODE_XTS) {
855 switch (keylen) {
856 case SEC_XTS_MIN_KEY_SIZE:
857 c_ctx->c_key_len = SEC_CKEY_128BIT;
858 break;
859 case SEC_XTS_MID_KEY_SIZE:
860 c_ctx->fallback = true;
861 break;
862 case SEC_XTS_MAX_KEY_SIZE:
863 c_ctx->c_key_len = SEC_CKEY_256BIT;
864 break;
865 default:
866 pr_err("hisi_sec2: xts mode key error!\n");
867 return -EINVAL;
868 }
869 } else {
870 if (c_ctx->c_alg == SEC_CALG_SM4 &&
871 keylen != AES_KEYSIZE_128) {
872 pr_err("hisi_sec2: sm4 key error!\n");
873 return -EINVAL;
874 } else {
875 switch (keylen) {
876 case AES_KEYSIZE_128:
877 c_ctx->c_key_len = SEC_CKEY_128BIT;
878 break;
879 case AES_KEYSIZE_192:
880 c_ctx->c_key_len = SEC_CKEY_192BIT;
881 break;
882 case AES_KEYSIZE_256:
883 c_ctx->c_key_len = SEC_CKEY_256BIT;
884 break;
885 default:
886 pr_err("hisi_sec2: aes key error!\n");
887 return -EINVAL;
888 }
889 }
890 }
891
892 return 0;
893 }
894
sec_skcipher_setkey(struct crypto_skcipher * tfm,const u8 * key,const u32 keylen,const enum sec_calg c_alg,const enum sec_cmode c_mode)895 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
896 const u32 keylen, const enum sec_calg c_alg,
897 const enum sec_cmode c_mode)
898 {
899 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
900 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
901 struct device *dev = ctx->dev;
902 int ret;
903
904 if (!ctx->qps)
905 goto set_soft_key;
906
907 if (c_mode == SEC_CMODE_XTS) {
908 ret = xts_verify_key(tfm, key, keylen);
909 if (ret) {
910 dev_err(dev, "xts mode key err!\n");
911 return ret;
912 }
913 }
914
915 c_ctx->c_alg = c_alg;
916 c_ctx->c_mode = c_mode;
917
918 switch (c_alg) {
919 case SEC_CALG_3DES:
920 ret = sec_skcipher_3des_setkey(tfm, key, keylen);
921 break;
922 case SEC_CALG_AES:
923 case SEC_CALG_SM4:
924 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
925 break;
926 default:
927 dev_err(dev, "sec c_alg err!\n");
928 return -EINVAL;
929 }
930
931 if (ret) {
932 dev_err(dev, "set sec key err!\n");
933 return ret;
934 }
935
936 memcpy(c_ctx->c_key, key, keylen);
937
938 set_soft_key:
939 ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen);
940 if (ret) {
941 dev_err(dev, "failed to set fallback skcipher key!\n");
942 return ret;
943 }
944
945 return 0;
946 }
947
948 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \
949 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\
950 u32 keylen) \
951 { \
952 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \
953 }
954
GEN_SEC_SETKEY_FUNC(aes_ecb,SEC_CALG_AES,SEC_CMODE_ECB)955 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)
956 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)
957 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)
958 GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR)
959 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)
960 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)
961 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)
962 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)
963 GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR)
964
965 static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
966 struct scatterlist *src)
967 {
968 struct aead_request *aead_req = req->aead_req.aead_req;
969 struct sec_cipher_req *c_req = &req->c_req;
970 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
971 struct sec_request_buf *buf = &req->buf;
972 struct device *dev = ctx->dev;
973 int copy_size, pbuf_length;
974 int req_id = req->req_id;
975 struct crypto_aead *tfm;
976 u8 *mac_offset, *pbuf;
977 size_t authsize;
978
979 if (ctx->alg_type == SEC_AEAD)
980 copy_size = aead_req->cryptlen + aead_req->assoclen;
981 else
982 copy_size = c_req->c_len;
983
984
985 pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf;
986 pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size);
987 if (unlikely(pbuf_length != copy_size)) {
988 dev_err(dev, "copy src data to pbuf error!\n");
989 return -EINVAL;
990 }
991 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
992 tfm = crypto_aead_reqtfm(aead_req);
993 authsize = crypto_aead_authsize(tfm);
994 mac_offset = pbuf + copy_size - authsize;
995 memcpy(req->aead_req.out_mac, mac_offset, authsize);
996 }
997
998 if (req->req_id < 0) {
999 buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL);
1000 if (unlikely(dma_mapping_error(dev, buf->in_dma)))
1001 return -ENOMEM;
1002
1003 buf->out_dma = buf->in_dma;
1004 return 0;
1005 }
1006
1007 req->in_dma = qp_ctx->res[req_id].pbuf_dma;
1008 c_req->c_out_dma = req->in_dma;
1009
1010 return 0;
1011 }
1012
sec_cipher_pbuf_unmap(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * dst)1013 static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
1014 struct scatterlist *dst)
1015 {
1016 struct aead_request *aead_req = req->aead_req.aead_req;
1017 struct sec_cipher_req *c_req = &req->c_req;
1018 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1019 struct sec_request_buf *buf = &req->buf;
1020 int copy_size, pbuf_length;
1021 int req_id = req->req_id;
1022
1023 if (ctx->alg_type == SEC_AEAD)
1024 copy_size = c_req->c_len + aead_req->assoclen;
1025 else
1026 copy_size = c_req->c_len;
1027
1028 if (req->req_id < 0)
1029 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size);
1030 else
1031 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf,
1032 copy_size);
1033 if (unlikely(pbuf_length != copy_size))
1034 dev_err(ctx->dev, "copy pbuf data to dst error!\n");
1035
1036 if (req->req_id < 0)
1037 dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL);
1038 }
1039
sec_aead_mac_init(struct sec_aead_req * req)1040 static int sec_aead_mac_init(struct sec_aead_req *req)
1041 {
1042 struct aead_request *aead_req = req->aead_req;
1043 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
1044 size_t authsize = crypto_aead_authsize(tfm);
1045 struct scatterlist *sgl = aead_req->src;
1046 u8 *mac_out = req->out_mac;
1047 size_t copy_size;
1048 off_t skip_size;
1049
1050 /* Copy input mac */
1051 skip_size = aead_req->assoclen + aead_req->cryptlen - authsize;
1052 copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, authsize, skip_size);
1053 if (unlikely(copy_size != authsize))
1054 return -EINVAL;
1055
1056 return 0;
1057 }
1058
fill_sg_to_hw_sge(struct scatterlist * sgl,struct sec_hw_sge * hw_sge)1059 static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge)
1060 {
1061 hw_sge->buf = sg_dma_address(sgl);
1062 hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
1063 hw_sge->page_ctrl = sg_virt(sgl);
1064 }
1065
sec_cipher_to_hw_sgl(struct device * dev,struct scatterlist * src,struct sec_hw_sgl * src_in,dma_addr_t * hw_sgl_dma,int dma_dir)1066 static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src,
1067 struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma,
1068 int dma_dir)
1069 {
1070 struct sec_hw_sge *curr_hw_sge = src_in->sge_entries;
1071 u32 i, sg_n, sg_n_mapped;
1072 struct scatterlist *sg;
1073 u32 sge_var = 0;
1074
1075 sg_n = sg_nents(src);
1076 sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir);
1077 if (unlikely(!sg_n_mapped)) {
1078 dev_err(dev, "dma mapping for SG error!\n");
1079 return -EINVAL;
1080 } else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) {
1081 dev_err(dev, "the number of entries in input scatterlist error!\n");
1082 dma_unmap_sg(dev, src, sg_n, dma_dir);
1083 return -EINVAL;
1084 }
1085
1086 for_each_sg(src, sg, sg_n_mapped, i) {
1087 fill_sg_to_hw_sge(sg, curr_hw_sge);
1088 curr_hw_sge++;
1089 sge_var++;
1090 }
1091
1092 src_in->entry_sum_in_sgl = cpu_to_le16(sge_var);
1093 src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM);
1094 src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM);
1095 *hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir);
1096 if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) {
1097 dma_unmap_sg(dev, src, sg_n, dma_dir);
1098 return -ENOMEM;
1099 }
1100
1101 return 0;
1102 }
1103
sec_cipher_put_hw_sgl(struct device * dev,struct scatterlist * src,dma_addr_t src_in,int dma_dir)1104 static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src,
1105 dma_addr_t src_in, int dma_dir)
1106 {
1107 dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir);
1108 dma_unmap_sg(dev, src, sg_nents(src), dma_dir);
1109 }
1110
sec_cipher_map_sgl(struct device * dev,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)1111 static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req,
1112 struct scatterlist *src, struct scatterlist *dst)
1113 {
1114 struct sec_hw_sgl *src_in = &req->buf.data_buf.in;
1115 struct sec_hw_sgl *dst_out = &req->buf.data_buf.out;
1116 int ret;
1117
1118 if (dst == src) {
1119 ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma,
1120 DMA_BIDIRECTIONAL);
1121 req->buf.out_dma = req->buf.in_dma;
1122 return ret;
1123 }
1124
1125 ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE);
1126 if (unlikely(ret))
1127 return ret;
1128
1129 ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma,
1130 DMA_FROM_DEVICE);
1131 if (unlikely(ret)) {
1132 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE);
1133 return ret;
1134 }
1135
1136 return 0;
1137 }
1138
sec_cipher_map_inner(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)1139 static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req,
1140 struct scatterlist *src, struct scatterlist *dst)
1141 {
1142 struct sec_cipher_req *c_req = &req->c_req;
1143 struct sec_aead_req *a_req = &req->aead_req;
1144 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1145 struct sec_alg_res *res = &qp_ctx->res[req->req_id];
1146 struct device *dev = ctx->dev;
1147 enum dma_data_direction src_direction;
1148 int ret;
1149
1150 if (req->use_pbuf) {
1151 c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET;
1152 c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET;
1153 if (ctx->alg_type == SEC_AEAD) {
1154 a_req->a_ivin = res->a_ivin;
1155 a_req->a_ivin_dma = res->a_ivin_dma;
1156 a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET;
1157 a_req->out_mac_dma = res->pbuf_dma +
1158 SEC_PBUF_MAC_OFFSET;
1159 }
1160 return sec_cipher_pbuf_map(ctx, req, src);
1161 }
1162
1163 c_req->c_ivin = res->c_ivin;
1164 c_req->c_ivin_dma = res->c_ivin_dma;
1165 if (ctx->alg_type == SEC_AEAD) {
1166 a_req->a_ivin = res->a_ivin;
1167 a_req->a_ivin_dma = res->a_ivin_dma;
1168 a_req->out_mac = res->out_mac;
1169 a_req->out_mac_dma = res->out_mac_dma;
1170 }
1171
1172 src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1173 req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
1174 qp_ctx->c_in_pool,
1175 req->req_id,
1176 &req->in_dma, src_direction);
1177 if (IS_ERR(req->in)) {
1178 dev_err(dev, "fail to dma map input sgl buffers!\n");
1179 return PTR_ERR(req->in);
1180 }
1181
1182 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
1183 ret = sec_aead_mac_init(a_req);
1184 if (unlikely(ret)) {
1185 dev_err(dev, "fail to init mac data for ICV!\n");
1186 hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction);
1187 return ret;
1188 }
1189 }
1190
1191 if (dst == src) {
1192 c_req->c_out = req->in;
1193 c_req->c_out_dma = req->in_dma;
1194 } else {
1195 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
1196 qp_ctx->c_out_pool,
1197 req->req_id,
1198 &c_req->c_out_dma,
1199 DMA_FROM_DEVICE);
1200
1201 if (IS_ERR(c_req->c_out)) {
1202 dev_err(dev, "fail to dma map output sgl buffers!\n");
1203 hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction);
1204 return PTR_ERR(c_req->c_out);
1205 }
1206 }
1207
1208 return 0;
1209 }
1210
sec_cipher_map(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)1211 static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
1212 struct scatterlist *src, struct scatterlist *dst)
1213 {
1214 struct sec_aead_req *a_req = &req->aead_req;
1215 struct sec_cipher_req *c_req = &req->c_req;
1216 bool is_aead = (ctx->alg_type == SEC_AEAD);
1217 struct device *dev = ctx->dev;
1218 int ret = -ENOMEM;
1219
1220 if (req->req_id >= 0)
1221 return sec_cipher_map_inner(ctx, req, src, dst);
1222
1223 c_req->c_ivin = c_req->c_ivin_buf;
1224 c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin,
1225 SEC_IV_SIZE, DMA_TO_DEVICE);
1226 if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma)))
1227 return -ENOMEM;
1228
1229 if (is_aead) {
1230 a_req->a_ivin = a_req->a_ivin_buf;
1231 a_req->out_mac = a_req->out_mac_buf;
1232 a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin,
1233 SEC_IV_SIZE, DMA_TO_DEVICE);
1234 if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma)))
1235 goto free_c_ivin_dma;
1236
1237 a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac,
1238 SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1239 if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma)))
1240 goto free_a_ivin_dma;
1241 }
1242 if (req->use_pbuf) {
1243 ret = sec_cipher_pbuf_map(ctx, req, src);
1244 if (unlikely(ret))
1245 goto free_out_mac_dma;
1246
1247 return 0;
1248 }
1249
1250 if (!c_req->encrypt && is_aead) {
1251 ret = sec_aead_mac_init(a_req);
1252 if (unlikely(ret)) {
1253 dev_err(dev, "fail to init mac data for ICV!\n");
1254 goto free_out_mac_dma;
1255 }
1256 }
1257
1258 ret = sec_cipher_map_sgl(dev, req, src, dst);
1259 if (unlikely(ret)) {
1260 dev_err(dev, "fail to dma map input sgl buffers!\n");
1261 goto free_out_mac_dma;
1262 }
1263
1264 return 0;
1265
1266 free_out_mac_dma:
1267 if (is_aead)
1268 dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1269 free_a_ivin_dma:
1270 if (is_aead)
1271 dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1272 free_c_ivin_dma:
1273 dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1274 return ret;
1275 }
1276
sec_cipher_unmap(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)1277 static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
1278 struct scatterlist *src, struct scatterlist *dst)
1279 {
1280 struct sec_aead_req *a_req = &req->aead_req;
1281 struct sec_cipher_req *c_req = &req->c_req;
1282 struct device *dev = ctx->dev;
1283
1284 if (req->req_id >= 0) {
1285 if (req->use_pbuf) {
1286 sec_cipher_pbuf_unmap(ctx, req, dst);
1287 } else {
1288 if (dst != src) {
1289 hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE);
1290 hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE);
1291 } else {
1292 hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL);
1293 }
1294 }
1295 return;
1296 }
1297
1298 if (req->use_pbuf) {
1299 sec_cipher_pbuf_unmap(ctx, req, dst);
1300 } else {
1301 if (dst != src) {
1302 sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE);
1303 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE);
1304 } else {
1305 sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL);
1306 }
1307 }
1308
1309 dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1310 if (ctx->alg_type == SEC_AEAD) {
1311 dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1312 dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1313 }
1314 }
1315
sec_skcipher_sgl_map(struct sec_ctx * ctx,struct sec_req * req)1316 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1317 {
1318 struct skcipher_request *sq = req->c_req.sk_req;
1319
1320 return sec_cipher_map(ctx, req, sq->src, sq->dst);
1321 }
1322
sec_skcipher_sgl_unmap(struct sec_ctx * ctx,struct sec_req * req)1323 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1324 {
1325 struct skcipher_request *sq = req->c_req.sk_req;
1326
1327 sec_cipher_unmap(ctx, req, sq->src, sq->dst);
1328 }
1329
sec_aead_aes_set_key(struct sec_cipher_ctx * c_ctx,struct crypto_authenc_keys * keys)1330 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,
1331 struct crypto_authenc_keys *keys)
1332 {
1333 switch (keys->enckeylen) {
1334 case AES_KEYSIZE_128:
1335 c_ctx->c_key_len = SEC_CKEY_128BIT;
1336 break;
1337 case AES_KEYSIZE_192:
1338 c_ctx->c_key_len = SEC_CKEY_192BIT;
1339 break;
1340 case AES_KEYSIZE_256:
1341 c_ctx->c_key_len = SEC_CKEY_256BIT;
1342 break;
1343 default:
1344 pr_err("hisi_sec2: aead aes key error!\n");
1345 return -EINVAL;
1346 }
1347 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);
1348
1349 return 0;
1350 }
1351
sec_aead_auth_set_key(struct sec_auth_ctx * ctx,struct crypto_authenc_keys * keys)1352 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
1353 struct crypto_authenc_keys *keys)
1354 {
1355 struct crypto_shash *hash_tfm = ctx->hash_tfm;
1356 int blocksize, digestsize, ret;
1357
1358 blocksize = crypto_shash_blocksize(hash_tfm);
1359 digestsize = crypto_shash_digestsize(hash_tfm);
1360 if (keys->authkeylen > blocksize) {
1361 ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,
1362 keys->authkeylen, ctx->a_key);
1363 if (ret) {
1364 pr_err("hisi_sec2: aead auth digest error!\n");
1365 return -EINVAL;
1366 }
1367 ctx->a_key_len = digestsize;
1368 } else {
1369 if (keys->authkeylen)
1370 memcpy(ctx->a_key, keys->authkey, keys->authkeylen);
1371 ctx->a_key_len = keys->authkeylen;
1372 }
1373
1374 return 0;
1375 }
1376
sec_aead_setauthsize(struct crypto_aead * aead,unsigned int authsize)1377 static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize)
1378 {
1379 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
1380 struct sec_ctx *ctx = crypto_tfm_ctx(tfm);
1381 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1382
1383 return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize);
1384 }
1385
sec_aead_fallback_setkey(struct sec_auth_ctx * a_ctx,struct crypto_aead * tfm,const u8 * key,unsigned int keylen)1386 static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx,
1387 struct crypto_aead *tfm, const u8 *key,
1388 unsigned int keylen)
1389 {
1390 crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK);
1391 crypto_aead_set_flags(a_ctx->fallback_aead_tfm,
1392 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
1393 return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen);
1394 }
1395
sec_aead_setkey(struct crypto_aead * tfm,const u8 * key,const u32 keylen,const enum sec_hash_alg a_alg,const enum sec_calg c_alg,const enum sec_cmode c_mode)1396 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
1397 const u32 keylen, const enum sec_hash_alg a_alg,
1398 const enum sec_calg c_alg,
1399 const enum sec_cmode c_mode)
1400 {
1401 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1402 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1403 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1404 struct device *dev = ctx->dev;
1405 struct crypto_authenc_keys keys;
1406 int ret;
1407
1408 if (!ctx->qps)
1409 return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
1410
1411 ctx->a_ctx.a_alg = a_alg;
1412 ctx->c_ctx.c_alg = c_alg;
1413 c_ctx->c_mode = c_mode;
1414
1415 if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) {
1416 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
1417 if (ret) {
1418 dev_err(dev, "set sec aes ccm cipher key err!\n");
1419 return ret;
1420 }
1421 memcpy(c_ctx->c_key, key, keylen);
1422
1423 return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
1424 }
1425
1426 ret = crypto_authenc_extractkeys(&keys, key, keylen);
1427 if (ret) {
1428 dev_err(dev, "sec extract aead keys err!\n");
1429 goto bad_key;
1430 }
1431
1432 ret = sec_aead_aes_set_key(c_ctx, &keys);
1433 if (ret) {
1434 dev_err(dev, "set sec cipher key err!\n");
1435 goto bad_key;
1436 }
1437
1438 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
1439 if (ret) {
1440 dev_err(dev, "set sec auth key err!\n");
1441 goto bad_key;
1442 }
1443
1444 ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
1445 if (ret) {
1446 dev_err(dev, "set sec fallback key err!\n");
1447 goto bad_key;
1448 }
1449
1450 return 0;
1451
1452 bad_key:
1453 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));
1454 return ret;
1455 }
1456
1457
1458 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, cmode) \
1459 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, u32 keylen) \
1460 { \
1461 return sec_aead_setkey(tfm, key, keylen, aalg, calg, cmode); \
1462 }
1463
GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1,SEC_A_HMAC_SHA1,SEC_CALG_AES,SEC_CMODE_CBC)1464 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_CMODE_CBC)
1465 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_CMODE_CBC)
1466 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_CMODE_CBC)
1467 GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_CMODE_CCM)
1468 GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_CMODE_GCM)
1469 GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_CMODE_CCM)
1470 GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_CMODE_GCM)
1471
1472 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1473 {
1474 struct aead_request *aq = req->aead_req.aead_req;
1475
1476 return sec_cipher_map(ctx, req, aq->src, aq->dst);
1477 }
1478
sec_aead_sgl_unmap(struct sec_ctx * ctx,struct sec_req * req)1479 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1480 {
1481 struct aead_request *aq = req->aead_req.aead_req;
1482
1483 sec_cipher_unmap(ctx, req, aq->src, aq->dst);
1484 }
1485
sec_request_transfer(struct sec_ctx * ctx,struct sec_req * req)1486 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)
1487 {
1488 int ret;
1489
1490 ret = ctx->req_op->buf_map(ctx, req);
1491 if (unlikely(ret))
1492 return ret;
1493
1494 ctx->req_op->do_transfer(ctx, req);
1495
1496 ret = ctx->req_op->bd_fill(ctx, req);
1497 if (unlikely(ret))
1498 goto unmap_req_buf;
1499
1500 return ret;
1501
1502 unmap_req_buf:
1503 ctx->req_op->buf_unmap(ctx, req);
1504 return ret;
1505 }
1506
sec_request_untransfer(struct sec_ctx * ctx,struct sec_req * req)1507 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)
1508 {
1509 ctx->req_op->buf_unmap(ctx, req);
1510 }
1511
sec_skcipher_copy_iv(struct sec_ctx * ctx,struct sec_req * req)1512 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
1513 {
1514 struct skcipher_request *sk_req = req->c_req.sk_req;
1515 struct sec_cipher_req *c_req = &req->c_req;
1516
1517 memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize);
1518 }
1519
sec_skcipher_bd_fill(struct sec_ctx * ctx,struct sec_req * req)1520 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1521 {
1522 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1523 struct sec_cipher_req *c_req = &req->c_req;
1524 struct sec_sqe *sec_sqe = &req->sec_sqe;
1525 u8 scene, sa_type, da_type;
1526 u8 bd_type, cipher;
1527 u8 de = 0;
1528
1529 memset(sec_sqe, 0, sizeof(struct sec_sqe));
1530
1531 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1532 sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1533 if (req->req_id < 0) {
1534 sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma);
1535 sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma);
1536 } else {
1537 sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1538 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1539 }
1540 if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr)
1541 de = 0x1 << SEC_DE_OFFSET;
1542
1543 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1544 SEC_CMODE_OFFSET);
1545 sec_sqe->type2.c_alg = c_ctx->c_alg;
1546 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1547 SEC_CKEY_OFFSET);
1548
1549 bd_type = SEC_BD_TYPE2;
1550 if (c_req->encrypt)
1551 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;
1552 else
1553 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;
1554 sec_sqe->type_cipher_auth = bd_type | cipher;
1555
1556 /* Set destination and source address type */
1557 if (req->use_pbuf) {
1558 sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET;
1559 da_type = SEC_PBUF << SEC_DST_SGL_OFFSET;
1560 } else {
1561 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;
1562 da_type = SEC_SGL << SEC_DST_SGL_OFFSET;
1563 }
1564
1565 sec_sqe->sdm_addr_type |= da_type;
1566 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
1567
1568 sec_sqe->sds_sa_type = (de | scene | sa_type);
1569
1570 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1571
1572 return 0;
1573 }
1574
sec_skcipher_bd_fill_v3(struct sec_ctx * ctx,struct sec_req * req)1575 static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1576 {
1577 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1578 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1579 struct sec_cipher_req *c_req = &req->c_req;
1580 u32 bd_param = 0;
1581 u16 cipher;
1582
1583 memset(sec_sqe3, 0, sizeof(struct sec_sqe3));
1584
1585 sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1586 sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1587 if (req->req_id < 0) {
1588 sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma);
1589 sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma);
1590 } else {
1591 sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1592 sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1593 }
1594 if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr)
1595 bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1596
1597 sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1598 c_ctx->c_mode;
1599 sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1600 SEC_CKEY_OFFSET_V3);
1601
1602 if (c_req->encrypt)
1603 cipher = SEC_CIPHER_ENC;
1604 else
1605 cipher = SEC_CIPHER_DEC;
1606 sec_sqe3->c_icv_key |= cpu_to_le16(cipher);
1607
1608 /* Set the CTR counter mode is 128bit rollover */
1609 sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER <<
1610 SEC_CTR_CNT_OFFSET);
1611
1612 if (req->use_pbuf) {
1613 bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3;
1614 bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3;
1615 } else {
1616 bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3;
1617 bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3;
1618 }
1619
1620 bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
1621
1622 bd_param |= SEC_BD_TYPE3;
1623 sec_sqe3->bd_param = cpu_to_le32(bd_param);
1624
1625 sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len);
1626 sec_sqe3->tag = cpu_to_le64((unsigned long)req);
1627
1628 return 0;
1629 }
1630
1631 /* increment counter (128-bit int) */
ctr_iv_inc(__u8 * counter,__u8 bits,__u32 nums)1632 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
1633 {
1634 do {
1635 --bits;
1636 nums += counter[bits];
1637 counter[bits] = nums & BITS_MASK;
1638 nums >>= BYTE_BITS;
1639 } while (bits && nums);
1640 }
1641
sec_update_iv(struct sec_req * req,enum sec_alg_type alg_type)1642 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
1643 {
1644 struct aead_request *aead_req = req->aead_req.aead_req;
1645 struct skcipher_request *sk_req = req->c_req.sk_req;
1646 u32 iv_size = req->ctx->c_ctx.ivsize;
1647 struct scatterlist *sgl;
1648 unsigned int cryptlen;
1649 size_t sz;
1650 u8 *iv;
1651
1652 if (alg_type == SEC_SKCIPHER) {
1653 sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src;
1654 iv = sk_req->iv;
1655 cryptlen = sk_req->cryptlen;
1656 } else {
1657 sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src;
1658 iv = aead_req->iv;
1659 cryptlen = aead_req->cryptlen;
1660 }
1661
1662 if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) {
1663 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
1664 cryptlen - iv_size);
1665 if (unlikely(sz != iv_size))
1666 dev_err(req->ctx->dev, "copy output iv error!\n");
1667 } else {
1668 sz = (cryptlen + iv_size - 1) / iv_size;
1669 ctr_iv_inc(iv, iv_size, sz);
1670 }
1671 }
1672
sec_skcipher_callback(struct sec_ctx * ctx,struct sec_req * req,int err)1673 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1674 int err)
1675 {
1676 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1677
1678 if (req->req_id >= 0)
1679 sec_free_req_id(req);
1680
1681 /* IV output at encrypto of CBC/CTR mode */
1682 if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1683 ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
1684 sec_update_iv(req, SEC_SKCIPHER);
1685
1686 crypto_request_complete(req->base, err);
1687 sec_alg_send_backlog(ctx, qp_ctx);
1688 }
1689
set_aead_auth_iv(struct sec_ctx * ctx,struct sec_req * req)1690 static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
1691 {
1692 struct aead_request *aead_req = req->aead_req.aead_req;
1693 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
1694 size_t authsize = crypto_aead_authsize(tfm);
1695 struct sec_aead_req *a_req = &req->aead_req;
1696 struct sec_cipher_req *c_req = &req->c_req;
1697 u32 data_size = aead_req->cryptlen;
1698 u8 flage = 0;
1699 u8 cm, cl;
1700
1701 /* the specification has been checked in aead_iv_demension_check() */
1702 cl = c_req->c_ivin[0] + 1;
1703 c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00;
1704 memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl);
1705 c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT;
1706
1707 /* the last 3bit is L' */
1708 flage |= c_req->c_ivin[0] & IV_CL_MASK;
1709
1710 /* the M' is bit3~bit5, the Flags is bit6 */
1711 cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM;
1712 flage |= cm << IV_CM_OFFSET;
1713 if (aead_req->assoclen)
1714 flage |= 0x01 << IV_FLAGS_OFFSET;
1715
1716 memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize);
1717 a_req->a_ivin[0] = flage;
1718
1719 /*
1720 * the last 32bit is counter's initial number,
1721 * but the nonce uses the first 16bit
1722 * the tail 16bit fill with the cipher length
1723 */
1724 if (!c_req->encrypt)
1725 data_size = aead_req->cryptlen - authsize;
1726
1727 a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] =
1728 data_size & IV_LAST_BYTE_MASK;
1729 data_size >>= IV_BYTE_OFFSET;
1730 a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] =
1731 data_size & IV_LAST_BYTE_MASK;
1732 }
1733
sec_aead_set_iv(struct sec_ctx * ctx,struct sec_req * req)1734 static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req)
1735 {
1736 struct aead_request *aead_req = req->aead_req.aead_req;
1737 struct sec_aead_req *a_req = &req->aead_req;
1738 struct sec_cipher_req *c_req = &req->c_req;
1739
1740 memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize);
1741
1742 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) {
1743 /*
1744 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter},
1745 * the counter must set to 0x01
1746 * CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length}
1747 */
1748 set_aead_auth_iv(ctx, req);
1749 } else if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) {
1750 /* GCM 12Byte Cipher_IV == Auth_IV */
1751 memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE);
1752 }
1753 }
1754
sec_auth_bd_fill_xcm(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe * sec_sqe)1755 static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir,
1756 struct sec_req *req, struct sec_sqe *sec_sqe)
1757 {
1758 struct sec_aead_req *a_req = &req->aead_req;
1759 struct aead_request *aq = a_req->aead_req;
1760 struct crypto_aead *tfm = crypto_aead_reqtfm(aq);
1761 size_t authsize = crypto_aead_authsize(tfm);
1762
1763 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1764 sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)authsize);
1765
1766 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1767 sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr;
1768 sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1769 sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET;
1770
1771 if (dir)
1772 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1773 else
1774 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1775
1776 sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen);
1777 sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0);
1778 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1779
1780 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1781 }
1782
sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe3 * sqe3)1783 static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir,
1784 struct sec_req *req, struct sec_sqe3 *sqe3)
1785 {
1786 struct sec_aead_req *a_req = &req->aead_req;
1787 struct aead_request *aq = a_req->aead_req;
1788 struct crypto_aead *tfm = crypto_aead_reqtfm(aq);
1789 size_t authsize = crypto_aead_authsize(tfm);
1790
1791 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1792 sqe3->c_icv_key |= cpu_to_le16((u16)authsize << SEC_MAC_OFFSET_V3);
1793
1794 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1795 sqe3->a_key_addr = sqe3->c_key_addr;
1796 sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1797 sqe3->auth_mac_key |= SEC_NO_AUTH;
1798
1799 if (dir)
1800 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1801 else
1802 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1803
1804 sqe3->a_len_key = cpu_to_le32(aq->assoclen);
1805 sqe3->auth_src_offset = cpu_to_le16(0x0);
1806 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1807 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1808 }
1809
sec_auth_bd_fill_ex(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe * sec_sqe)1810 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,
1811 struct sec_req *req, struct sec_sqe *sec_sqe)
1812 {
1813 struct sec_aead_req *a_req = &req->aead_req;
1814 struct sec_cipher_req *c_req = &req->c_req;
1815 struct aead_request *aq = a_req->aead_req;
1816 struct crypto_aead *tfm = crypto_aead_reqtfm(aq);
1817 size_t authsize = crypto_aead_authsize(tfm);
1818
1819 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);
1820
1821 sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize));
1822
1823 sec_sqe->type2.mac_key_alg |=
1824 cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET);
1825
1826 sec_sqe->type2.mac_key_alg |=
1827 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);
1828
1829 if (dir) {
1830 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;
1831 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1832 } else {
1833 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET;
1834 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1835 }
1836 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);
1837
1838 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1839
1840 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1841 }
1842
sec_aead_bd_fill(struct sec_ctx * ctx,struct sec_req * req)1843 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1844 {
1845 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1846 struct sec_sqe *sec_sqe = &req->sec_sqe;
1847 int ret;
1848
1849 ret = sec_skcipher_bd_fill(ctx, req);
1850 if (unlikely(ret)) {
1851 dev_err(ctx->dev, "skcipher bd fill is error!\n");
1852 return ret;
1853 }
1854
1855 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1856 ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1857 sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1858 else
1859 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1860
1861 return 0;
1862 }
1863
sec_auth_bd_fill_ex_v3(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe3 * sqe3)1864 static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir,
1865 struct sec_req *req, struct sec_sqe3 *sqe3)
1866 {
1867 struct sec_aead_req *a_req = &req->aead_req;
1868 struct sec_cipher_req *c_req = &req->c_req;
1869 struct aead_request *aq = a_req->aead_req;
1870 struct crypto_aead *tfm = crypto_aead_reqtfm(aq);
1871 size_t authsize = crypto_aead_authsize(tfm);
1872
1873 sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma);
1874
1875 sqe3->auth_mac_key |=
1876 cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3);
1877
1878 sqe3->auth_mac_key |=
1879 cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3);
1880
1881 sqe3->auth_mac_key |=
1882 cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3);
1883
1884 if (dir) {
1885 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);
1886 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1887 } else {
1888 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2);
1889 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1890 }
1891 sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen);
1892
1893 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1894
1895 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1896 }
1897
sec_aead_bd_fill_v3(struct sec_ctx * ctx,struct sec_req * req)1898 static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1899 {
1900 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1901 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1902 int ret;
1903
1904 ret = sec_skcipher_bd_fill_v3(ctx, req);
1905 if (unlikely(ret)) {
1906 dev_err(ctx->dev, "skcipher bd3 fill is error!\n");
1907 return ret;
1908 }
1909
1910 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1911 ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1912 sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt,
1913 req, sec_sqe3);
1914 else
1915 sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt,
1916 req, sec_sqe3);
1917
1918 return 0;
1919 }
1920
sec_aead_callback(struct sec_ctx * c,struct sec_req * req,int err)1921 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
1922 {
1923 struct aead_request *a_req = req->aead_req.aead_req;
1924 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
1925 size_t authsize = crypto_aead_authsize(tfm);
1926 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1927 size_t sz;
1928
1929 if (!err && req->c_req.encrypt) {
1930 if (c->c_ctx.c_mode == SEC_CMODE_CBC)
1931 sec_update_iv(req, SEC_AEAD);
1932
1933 sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac,
1934 authsize, a_req->cryptlen + a_req->assoclen);
1935 if (unlikely(sz != authsize)) {
1936 dev_err(c->dev, "copy out mac err!\n");
1937 err = -EINVAL;
1938 }
1939 }
1940
1941 if (req->req_id >= 0)
1942 sec_free_req_id(req);
1943
1944 crypto_request_complete(req->base, err);
1945 sec_alg_send_backlog(c, qp_ctx);
1946 }
1947
sec_request_uninit(struct sec_req * req)1948 static void sec_request_uninit(struct sec_req *req)
1949 {
1950 if (req->req_id >= 0)
1951 sec_free_req_id(req);
1952 }
1953
sec_request_init(struct sec_ctx * ctx,struct sec_req * req)1954 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1955 {
1956 struct sec_qp_ctx *qp_ctx;
1957 int i = 0;
1958
1959 do {
1960 qp_ctx = &ctx->qp_ctx[i];
1961 req->req_id = sec_alloc_req_id(req, qp_ctx);
1962 } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num);
1963
1964 req->qp_ctx = qp_ctx;
1965
1966 return 0;
1967 }
1968
sec_process(struct sec_ctx * ctx,struct sec_req * req)1969 static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1970 {
1971 int ret;
1972
1973 ret = sec_request_init(ctx, req);
1974 if (unlikely(ret))
1975 return ret;
1976
1977 ret = sec_request_transfer(ctx, req);
1978 if (unlikely(ret))
1979 goto err_uninit_req;
1980
1981 /* Output IV as decrypto */
1982 if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1983 ctx->c_ctx.c_mode == SEC_CMODE_CTR))
1984 sec_update_iv(req, ctx->alg_type);
1985
1986 ret = ctx->req_op->bd_send(ctx, req);
1987 if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) {
1988 dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1989 goto err_send_req;
1990 }
1991
1992 return ret;
1993
1994 err_send_req:
1995 /* As failing, restore the IV from user */
1996 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
1997 if (ctx->alg_type == SEC_SKCIPHER)
1998 memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin,
1999 ctx->c_ctx.ivsize);
2000 else
2001 memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin,
2002 ctx->c_ctx.ivsize);
2003 }
2004
2005 sec_request_untransfer(ctx, req);
2006
2007 err_uninit_req:
2008 sec_request_uninit(req);
2009 if (ctx->alg_type == SEC_AEAD)
2010 ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req,
2011 req->c_req.encrypt);
2012 else
2013 ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req,
2014 req->c_req.encrypt);
2015 return ret;
2016 }
2017
2018 static const struct sec_req_op sec_skcipher_req_ops = {
2019 .buf_map = sec_skcipher_sgl_map,
2020 .buf_unmap = sec_skcipher_sgl_unmap,
2021 .do_transfer = sec_skcipher_copy_iv,
2022 .bd_fill = sec_skcipher_bd_fill,
2023 .bd_send = sec_bd_send,
2024 .callback = sec_skcipher_callback,
2025 .process = sec_process,
2026 };
2027
2028 static const struct sec_req_op sec_aead_req_ops = {
2029 .buf_map = sec_aead_sgl_map,
2030 .buf_unmap = sec_aead_sgl_unmap,
2031 .do_transfer = sec_aead_set_iv,
2032 .bd_fill = sec_aead_bd_fill,
2033 .bd_send = sec_bd_send,
2034 .callback = sec_aead_callback,
2035 .process = sec_process,
2036 };
2037
2038 static const struct sec_req_op sec_skcipher_req_ops_v3 = {
2039 .buf_map = sec_skcipher_sgl_map,
2040 .buf_unmap = sec_skcipher_sgl_unmap,
2041 .do_transfer = sec_skcipher_copy_iv,
2042 .bd_fill = sec_skcipher_bd_fill_v3,
2043 .bd_send = sec_bd_send,
2044 .callback = sec_skcipher_callback,
2045 .process = sec_process,
2046 };
2047
2048 static const struct sec_req_op sec_aead_req_ops_v3 = {
2049 .buf_map = sec_aead_sgl_map,
2050 .buf_unmap = sec_aead_sgl_unmap,
2051 .do_transfer = sec_aead_set_iv,
2052 .bd_fill = sec_aead_bd_fill_v3,
2053 .bd_send = sec_bd_send,
2054 .callback = sec_aead_callback,
2055 .process = sec_process,
2056 };
2057
sec_skcipher_ctx_init(struct crypto_skcipher * tfm)2058 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)
2059 {
2060 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2061 int ret;
2062
2063 ret = sec_skcipher_init(tfm);
2064 if (ret)
2065 return ret;
2066
2067 if (!ctx->qps)
2068 return 0;
2069
2070 if (ctx->sec->qm.ver < QM_HW_V3) {
2071 ctx->type_supported = SEC_BD_TYPE2;
2072 ctx->req_op = &sec_skcipher_req_ops;
2073 } else {
2074 ctx->type_supported = SEC_BD_TYPE3;
2075 ctx->req_op = &sec_skcipher_req_ops_v3;
2076 }
2077
2078 return 0;
2079 }
2080
sec_skcipher_ctx_exit(struct crypto_skcipher * tfm)2081 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)
2082 {
2083 sec_skcipher_uninit(tfm);
2084 }
2085
sec_aead_init(struct crypto_aead * tfm)2086 static int sec_aead_init(struct crypto_aead *tfm)
2087 {
2088 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2089 int ret;
2090
2091 crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req));
2092 ctx->alg_type = SEC_AEAD;
2093 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
2094 if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
2095 ctx->c_ctx.ivsize > SEC_IV_SIZE) {
2096 pr_err("get error aead iv size!\n");
2097 return -EINVAL;
2098 }
2099
2100 ret = sec_ctx_base_init(ctx);
2101 if (ret)
2102 return ret;
2103 if (ctx->sec->qm.ver < QM_HW_V3) {
2104 ctx->type_supported = SEC_BD_TYPE2;
2105 ctx->req_op = &sec_aead_req_ops;
2106 } else {
2107 ctx->type_supported = SEC_BD_TYPE3;
2108 ctx->req_op = &sec_aead_req_ops_v3;
2109 }
2110
2111 ret = sec_auth_init(ctx);
2112 if (ret)
2113 goto err_auth_init;
2114
2115 ret = sec_cipher_init(ctx);
2116 if (ret)
2117 goto err_cipher_init;
2118
2119 return ret;
2120
2121 err_cipher_init:
2122 sec_auth_uninit(ctx);
2123 err_auth_init:
2124 sec_ctx_base_uninit(ctx);
2125 return ret;
2126 }
2127
sec_aead_exit(struct crypto_aead * tfm)2128 static void sec_aead_exit(struct crypto_aead *tfm)
2129 {
2130 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2131
2132 sec_cipher_uninit(ctx);
2133 sec_auth_uninit(ctx);
2134 sec_ctx_base_uninit(ctx);
2135 }
2136
sec_aead_ctx_init(struct crypto_aead * tfm,const char * hash_name)2137 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
2138 {
2139 struct aead_alg *alg = crypto_aead_alg(tfm);
2140 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2141 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
2142 const char *aead_name = alg->base.cra_name;
2143 int ret;
2144
2145 ret = sec_aead_init(tfm);
2146 if (ret && ret != -ENODEV) {
2147 pr_err("hisi_sec2: aead init error!\n");
2148 return ret;
2149 }
2150
2151 a_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
2152 if (IS_ERR(a_ctx->hash_tfm)) {
2153 dev_err(ctx->dev, "aead alloc shash error!\n");
2154 sec_aead_exit(tfm);
2155 return PTR_ERR(a_ctx->hash_tfm);
2156 }
2157
2158 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,
2159 CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC);
2160 if (IS_ERR(a_ctx->fallback_aead_tfm)) {
2161 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");
2162 crypto_free_shash(ctx->a_ctx.hash_tfm);
2163 sec_aead_exit(tfm);
2164 return PTR_ERR(a_ctx->fallback_aead_tfm);
2165 }
2166
2167 return 0;
2168 }
2169
sec_aead_ctx_exit(struct crypto_aead * tfm)2170 static void sec_aead_ctx_exit(struct crypto_aead *tfm)
2171 {
2172 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2173
2174 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);
2175 crypto_free_shash(ctx->a_ctx.hash_tfm);
2176 sec_aead_exit(tfm);
2177 }
2178
sec_aead_xcm_ctx_init(struct crypto_aead * tfm)2179 static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm)
2180 {
2181 struct aead_alg *alg = crypto_aead_alg(tfm);
2182 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2183 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
2184 const char *aead_name = alg->base.cra_name;
2185 int ret;
2186
2187 ret = sec_aead_init(tfm);
2188 if (ret && ret != -ENODEV) {
2189 dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n");
2190 return ret;
2191 }
2192
2193 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,
2194 CRYPTO_ALG_NEED_FALLBACK |
2195 CRYPTO_ALG_ASYNC);
2196 if (IS_ERR(a_ctx->fallback_aead_tfm)) {
2197 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");
2198 sec_aead_exit(tfm);
2199 return PTR_ERR(a_ctx->fallback_aead_tfm);
2200 }
2201
2202 return 0;
2203 }
2204
sec_aead_xcm_ctx_exit(struct crypto_aead * tfm)2205 static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm)
2206 {
2207 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2208
2209 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);
2210 sec_aead_exit(tfm);
2211 }
2212
sec_aead_sha1_ctx_init(struct crypto_aead * tfm)2213 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)
2214 {
2215 return sec_aead_ctx_init(tfm, "sha1");
2216 }
2217
sec_aead_sha256_ctx_init(struct crypto_aead * tfm)2218 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)
2219 {
2220 return sec_aead_ctx_init(tfm, "sha256");
2221 }
2222
sec_aead_sha512_ctx_init(struct crypto_aead * tfm)2223 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
2224 {
2225 return sec_aead_ctx_init(tfm, "sha512");
2226 }
2227
sec_skcipher_cryptlen_check(struct sec_ctx * ctx,struct sec_req * sreq)2228 static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq)
2229 {
2230 u32 cryptlen = sreq->c_req.sk_req->cryptlen;
2231 struct device *dev = ctx->dev;
2232 u8 c_mode = ctx->c_ctx.c_mode;
2233 int ret = 0;
2234
2235 switch (c_mode) {
2236 case SEC_CMODE_XTS:
2237 if (unlikely(cryptlen < AES_BLOCK_SIZE)) {
2238 dev_err(dev, "skcipher XTS mode input length error!\n");
2239 ret = -EINVAL;
2240 }
2241 break;
2242 case SEC_CMODE_ECB:
2243 case SEC_CMODE_CBC:
2244 if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {
2245 dev_err(dev, "skcipher AES input length error!\n");
2246 ret = -EINVAL;
2247 }
2248 break;
2249 case SEC_CMODE_CTR:
2250 break;
2251 default:
2252 ret = -EINVAL;
2253 }
2254
2255 return ret;
2256 }
2257
sec_skcipher_param_check(struct sec_ctx * ctx,struct sec_req * sreq,bool * need_fallback)2258 static int sec_skcipher_param_check(struct sec_ctx *ctx,
2259 struct sec_req *sreq, bool *need_fallback)
2260 {
2261 struct skcipher_request *sk_req = sreq->c_req.sk_req;
2262 struct device *dev = ctx->dev;
2263 u8 c_alg = ctx->c_ctx.c_alg;
2264
2265 if (unlikely(!sk_req->src || !sk_req->dst)) {
2266 dev_err(dev, "skcipher input param error!\n");
2267 return -EINVAL;
2268 }
2269
2270 if (sk_req->cryptlen > MAX_INPUT_DATA_LEN)
2271 *need_fallback = true;
2272
2273 sreq->c_req.c_len = sk_req->cryptlen;
2274
2275 if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ)
2276 sreq->use_pbuf = true;
2277 else
2278 sreq->use_pbuf = false;
2279
2280 if (c_alg == SEC_CALG_3DES) {
2281 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {
2282 dev_err(dev, "skcipher 3des input length error!\n");
2283 return -EINVAL;
2284 }
2285 return 0;
2286 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
2287 return sec_skcipher_cryptlen_check(ctx, sreq);
2288 }
2289
2290 dev_err(dev, "skcipher algorithm error!\n");
2291
2292 return -EINVAL;
2293 }
2294
sec_skcipher_soft_crypto(struct sec_ctx * ctx,struct skcipher_request * sreq,bool encrypt)2295 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
2296 struct skcipher_request *sreq, bool encrypt)
2297 {
2298 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
2299 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm);
2300 struct device *dev = ctx->dev;
2301 int ret;
2302
2303 if (!c_ctx->fbtfm) {
2304 dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n");
2305 return -EINVAL;
2306 }
2307
2308 skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm);
2309
2310 /* software need sync mode to do crypto */
2311 skcipher_request_set_callback(subreq, sreq->base.flags,
2312 NULL, NULL);
2313 skcipher_request_set_crypt(subreq, sreq->src, sreq->dst,
2314 sreq->cryptlen, sreq->iv);
2315 if (encrypt)
2316 ret = crypto_skcipher_encrypt(subreq);
2317 else
2318 ret = crypto_skcipher_decrypt(subreq);
2319
2320 skcipher_request_zero(subreq);
2321
2322 return ret;
2323 }
2324
sec_skcipher_crypto(struct skcipher_request * sk_req,bool encrypt)2325 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2326 {
2327 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2328 struct sec_req *req = skcipher_request_ctx_dma(sk_req);
2329 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2330 bool need_fallback = false;
2331 int ret;
2332
2333 if (!ctx->qps)
2334 goto soft_crypto;
2335
2336 if (!sk_req->cryptlen) {
2337 if (ctx->c_ctx.c_mode == SEC_CMODE_XTS)
2338 return -EINVAL;
2339 return 0;
2340 }
2341
2342 req->flag = sk_req->base.flags;
2343 req->c_req.sk_req = sk_req;
2344 req->c_req.encrypt = encrypt;
2345 req->ctx = ctx;
2346 req->base = &sk_req->base;
2347
2348 ret = sec_skcipher_param_check(ctx, req, &need_fallback);
2349 if (unlikely(ret))
2350 return -EINVAL;
2351
2352 if (unlikely(ctx->c_ctx.fallback || need_fallback))
2353 goto soft_crypto;
2354
2355 return ctx->req_op->process(ctx, req);
2356
2357 soft_crypto:
2358 return sec_skcipher_soft_crypto(ctx, sk_req, encrypt);
2359 }
2360
sec_skcipher_encrypt(struct skcipher_request * sk_req)2361 static int sec_skcipher_encrypt(struct skcipher_request *sk_req)
2362 {
2363 return sec_skcipher_crypto(sk_req, true);
2364 }
2365
sec_skcipher_decrypt(struct skcipher_request * sk_req)2366 static int sec_skcipher_decrypt(struct skcipher_request *sk_req)
2367 {
2368 return sec_skcipher_crypto(sk_req, false);
2369 }
2370
2371 #define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \
2372 sec_min_key_size, sec_max_key_size, blk_size, iv_size)\
2373 {\
2374 .base = {\
2375 .cra_name = sec_cra_name,\
2376 .cra_driver_name = "hisi_sec_"sec_cra_name,\
2377 .cra_priority = SEC_PRIORITY,\
2378 .cra_flags = CRYPTO_ALG_ASYNC |\
2379 CRYPTO_ALG_NEED_FALLBACK,\
2380 .cra_blocksize = blk_size,\
2381 .cra_ctxsize = sizeof(struct sec_ctx),\
2382 .cra_module = THIS_MODULE,\
2383 },\
2384 .init = sec_skcipher_ctx_init,\
2385 .exit = sec_skcipher_ctx_exit,\
2386 .setkey = sec_set_key,\
2387 .decrypt = sec_skcipher_decrypt,\
2388 .encrypt = sec_skcipher_encrypt,\
2389 .min_keysize = sec_min_key_size,\
2390 .max_keysize = sec_max_key_size,\
2391 .ivsize = iv_size,\
2392 }
2393
2394 static struct sec_skcipher sec_skciphers[] = {
2395 {
2396 .alg_msk = BIT(0),
2397 .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE,
2398 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0),
2399 },
2400 {
2401 .alg_msk = BIT(1),
2402 .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE,
2403 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2404 },
2405 {
2406 .alg_msk = BIT(2),
2407 .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE,
2408 AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2409 },
2410 {
2411 .alg_msk = BIT(3),
2412 .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE,
2413 SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2414 },
2415 {
2416 .alg_msk = BIT(12),
2417 .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE,
2418 AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2419 },
2420 {
2421 .alg_msk = BIT(13),
2422 .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE,
2423 AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2424 },
2425 {
2426 .alg_msk = BIT(14),
2427 .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE,
2428 SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2429 },
2430 {
2431 .alg_msk = BIT(23),
2432 .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE,
2433 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0),
2434 },
2435 {
2436 .alg_msk = BIT(24),
2437 .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE,
2438 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE,
2439 DES3_EDE_BLOCK_SIZE),
2440 },
2441 };
2442
aead_iv_demension_check(struct aead_request * aead_req)2443 static int aead_iv_demension_check(struct aead_request *aead_req)
2444 {
2445 u8 cl;
2446
2447 cl = aead_req->iv[0] + 1;
2448 if (cl < IV_CL_MIN || cl > IV_CL_MAX)
2449 return -EINVAL;
2450
2451 if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl))
2452 return -EOVERFLOW;
2453
2454 return 0;
2455 }
2456
sec_aead_spec_check(struct sec_ctx * ctx,struct sec_req * sreq)2457 static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq)
2458 {
2459 struct aead_request *req = sreq->aead_req.aead_req;
2460 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2461 size_t sz = crypto_aead_authsize(tfm);
2462 u8 c_mode = ctx->c_ctx.c_mode;
2463 int ret;
2464
2465 if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len))
2466 return -EINVAL;
2467
2468 if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN ||
2469 req->assoclen > SEC_MAX_AAD_LEN))
2470 return -EINVAL;
2471
2472 if (c_mode == SEC_CMODE_CCM) {
2473 if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN))
2474 return -EINVAL;
2475
2476 ret = aead_iv_demension_check(req);
2477 if (unlikely(ret))
2478 return -EINVAL;
2479 } else if (c_mode == SEC_CMODE_CBC) {
2480 if (unlikely(sz & WORD_MASK))
2481 return -EINVAL;
2482 if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK))
2483 return -EINVAL;
2484 } else if (c_mode == SEC_CMODE_GCM) {
2485 if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ))
2486 return -EINVAL;
2487 }
2488
2489 return 0;
2490 }
2491
sec_aead_param_check(struct sec_ctx * ctx,struct sec_req * sreq,bool * need_fallback)2492 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback)
2493 {
2494 struct aead_request *req = sreq->aead_req.aead_req;
2495 struct device *dev = ctx->dev;
2496 u8 c_alg = ctx->c_ctx.c_alg;
2497
2498 if (unlikely(!req->src || !req->dst)) {
2499 dev_err(dev, "aead input param error!\n");
2500 return -EINVAL;
2501 }
2502
2503 if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC &&
2504 sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
2505 dev_err(dev, "aead cbc mode input data length error!\n");
2506 return -EINVAL;
2507 }
2508
2509 /* Support AES or SM4 */
2510 if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) {
2511 dev_err(dev, "aead crypto alg error!\n");
2512 return -EINVAL;
2513 }
2514
2515 if (unlikely(sec_aead_spec_check(ctx, sreq))) {
2516 *need_fallback = true;
2517 return -EINVAL;
2518 }
2519
2520 if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <=
2521 SEC_PBUF_SZ)
2522 sreq->use_pbuf = true;
2523 else
2524 sreq->use_pbuf = false;
2525
2526 return 0;
2527 }
2528
sec_aead_soft_crypto(struct sec_ctx * ctx,struct aead_request * aead_req,bool encrypt)2529 static int sec_aead_soft_crypto(struct sec_ctx *ctx,
2530 struct aead_request *aead_req,
2531 bool encrypt)
2532 {
2533 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
2534 struct aead_request *subreq;
2535 int ret;
2536
2537 subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL);
2538 if (!subreq)
2539 return -ENOMEM;
2540
2541 aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);
2542 aead_request_set_callback(subreq, aead_req->base.flags,
2543 aead_req->base.complete, aead_req->base.data);
2544 aead_request_set_crypt(subreq, aead_req->src, aead_req->dst,
2545 aead_req->cryptlen, aead_req->iv);
2546 aead_request_set_ad(subreq, aead_req->assoclen);
2547
2548 if (encrypt)
2549 ret = crypto_aead_encrypt(subreq);
2550 else
2551 ret = crypto_aead_decrypt(subreq);
2552 aead_request_free(subreq);
2553
2554 return ret;
2555 }
2556
sec_aead_crypto(struct aead_request * a_req,bool encrypt)2557 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
2558 {
2559 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
2560 struct sec_req *req = aead_request_ctx_dma(a_req);
2561 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2562 size_t sz = crypto_aead_authsize(tfm);
2563 bool need_fallback = false;
2564 int ret;
2565
2566 if (!ctx->qps)
2567 goto soft_crypto;
2568
2569 req->flag = a_req->base.flags;
2570 req->aead_req.aead_req = a_req;
2571 req->c_req.encrypt = encrypt;
2572 req->ctx = ctx;
2573 req->base = &a_req->base;
2574 req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz);
2575
2576 ret = sec_aead_param_check(ctx, req, &need_fallback);
2577 if (unlikely(ret)) {
2578 if (need_fallback)
2579 goto soft_crypto;
2580 return -EINVAL;
2581 }
2582
2583 return ctx->req_op->process(ctx, req);
2584
2585 soft_crypto:
2586 return sec_aead_soft_crypto(ctx, a_req, encrypt);
2587 }
2588
sec_aead_encrypt(struct aead_request * a_req)2589 static int sec_aead_encrypt(struct aead_request *a_req)
2590 {
2591 return sec_aead_crypto(a_req, true);
2592 }
2593
sec_aead_decrypt(struct aead_request * a_req)2594 static int sec_aead_decrypt(struct aead_request *a_req)
2595 {
2596 return sec_aead_crypto(a_req, false);
2597 }
2598
2599 #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\
2600 ctx_exit, blk_size, iv_size, max_authsize)\
2601 {\
2602 .base = {\
2603 .cra_name = sec_cra_name,\
2604 .cra_driver_name = "hisi_sec_"sec_cra_name,\
2605 .cra_priority = SEC_PRIORITY,\
2606 .cra_flags = CRYPTO_ALG_ASYNC |\
2607 CRYPTO_ALG_NEED_FALLBACK,\
2608 .cra_blocksize = blk_size,\
2609 .cra_ctxsize = sizeof(struct sec_ctx),\
2610 .cra_module = THIS_MODULE,\
2611 },\
2612 .init = ctx_init,\
2613 .exit = ctx_exit,\
2614 .setkey = sec_set_key,\
2615 .setauthsize = sec_aead_setauthsize,\
2616 .decrypt = sec_aead_decrypt,\
2617 .encrypt = sec_aead_encrypt,\
2618 .ivsize = iv_size,\
2619 .maxauthsize = max_authsize,\
2620 }
2621
2622 static struct sec_aead sec_aeads[] = {
2623 {
2624 .alg_msk = BIT(6),
2625 .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init,
2626 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2627 AES_BLOCK_SIZE),
2628 },
2629 {
2630 .alg_msk = BIT(7),
2631 .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init,
2632 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2633 AES_BLOCK_SIZE),
2634 },
2635 {
2636 .alg_msk = BIT(17),
2637 .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init,
2638 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2639 AES_BLOCK_SIZE),
2640 },
2641 {
2642 .alg_msk = BIT(18),
2643 .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init,
2644 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2645 AES_BLOCK_SIZE),
2646 },
2647 {
2648 .alg_msk = BIT(43),
2649 .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1,
2650 sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2651 AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),
2652 },
2653 {
2654 .alg_msk = BIT(44),
2655 .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256,
2656 sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2657 AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),
2658 },
2659 {
2660 .alg_msk = BIT(45),
2661 .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512,
2662 sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2663 AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
2664 },
2665 };
2666
sec_unregister_skcipher(u64 alg_mask,int end)2667 static void sec_unregister_skcipher(u64 alg_mask, int end)
2668 {
2669 int i;
2670
2671 for (i = 0; i < end; i++)
2672 if (sec_skciphers[i].alg_msk & alg_mask)
2673 crypto_unregister_skcipher(&sec_skciphers[i].alg);
2674 }
2675
sec_register_skcipher(u64 alg_mask)2676 static int sec_register_skcipher(u64 alg_mask)
2677 {
2678 int i, ret, count;
2679
2680 count = ARRAY_SIZE(sec_skciphers);
2681
2682 for (i = 0; i < count; i++) {
2683 if (!(sec_skciphers[i].alg_msk & alg_mask))
2684 continue;
2685
2686 ret = crypto_register_skcipher(&sec_skciphers[i].alg);
2687 if (ret)
2688 goto err;
2689 }
2690
2691 return 0;
2692
2693 err:
2694 sec_unregister_skcipher(alg_mask, i);
2695
2696 return ret;
2697 }
2698
sec_unregister_aead(u64 alg_mask,int end)2699 static void sec_unregister_aead(u64 alg_mask, int end)
2700 {
2701 int i;
2702
2703 for (i = 0; i < end; i++)
2704 if (sec_aeads[i].alg_msk & alg_mask)
2705 crypto_unregister_aead(&sec_aeads[i].alg);
2706 }
2707
sec_register_aead(u64 alg_mask)2708 static int sec_register_aead(u64 alg_mask)
2709 {
2710 int i, ret, count;
2711
2712 count = ARRAY_SIZE(sec_aeads);
2713
2714 for (i = 0; i < count; i++) {
2715 if (!(sec_aeads[i].alg_msk & alg_mask))
2716 continue;
2717
2718 ret = crypto_register_aead(&sec_aeads[i].alg);
2719 if (ret)
2720 goto err;
2721 }
2722
2723 return 0;
2724
2725 err:
2726 sec_unregister_aead(alg_mask, i);
2727
2728 return ret;
2729 }
2730
sec_register_to_crypto(struct hisi_qm * qm)2731 int sec_register_to_crypto(struct hisi_qm *qm)
2732 {
2733 u64 alg_mask;
2734 int ret = 0;
2735
2736 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB,
2737 SEC_DRV_ALG_BITMAP_LOW_TB);
2738
2739 mutex_lock(&sec_algs_lock);
2740 if (sec_available_devs) {
2741 sec_available_devs++;
2742 goto unlock;
2743 }
2744
2745 ret = sec_register_skcipher(alg_mask);
2746 if (ret)
2747 goto unlock;
2748
2749 ret = sec_register_aead(alg_mask);
2750 if (ret)
2751 goto unreg_skcipher;
2752
2753 sec_available_devs++;
2754 mutex_unlock(&sec_algs_lock);
2755
2756 return 0;
2757
2758 unreg_skcipher:
2759 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2760 unlock:
2761 mutex_unlock(&sec_algs_lock);
2762 return ret;
2763 }
2764
sec_unregister_from_crypto(struct hisi_qm * qm)2765 void sec_unregister_from_crypto(struct hisi_qm *qm)
2766 {
2767 u64 alg_mask;
2768
2769 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB,
2770 SEC_DRV_ALG_BITMAP_LOW_TB);
2771
2772 mutex_lock(&sec_algs_lock);
2773 if (--sec_available_devs)
2774 goto unlock;
2775
2776 sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads));
2777 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2778
2779 unlock:
2780 mutex_unlock(&sec_algs_lock);
2781 }
2782