1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/dh.h>
5 #include <crypto/ecc_curve.h>
6 #include <crypto/ecdh.h>
7 #include <crypto/rng.h>
8 #include <crypto/internal/akcipher.h>
9 #include <crypto/internal/kpp.h>
10 #include <crypto/internal/rsa.h>
11 #include <crypto/kpp.h>
12 #include <crypto/scatterwalk.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/fips.h>
15 #include <linux/module.h>
16 #include <linux/time.h>
17 #include "hpre.h"
18
19 struct hpre_ctx;
20
21 #define HPRE_CRYPTO_ALG_PRI 1000
22 #define HPRE_ALIGN_SZ 64
23 #define HPRE_BITS_2_BYTES_SHIFT 3
24 #define HPRE_RSA_512BITS_KSZ 64
25 #define HPRE_RSA_1536BITS_KSZ 192
26 #define HPRE_CRT_PRMS 5
27 #define HPRE_CRT_Q 2
28 #define HPRE_CRT_P 3
29 #define HPRE_CRT_INV 4
30 #define HPRE_DH_G_FLAG 0x02
31 #define HPRE_TRY_SEND_TIMES 100
32 #define HPRE_INVLD_REQ_ID (-1)
33
34 #define HPRE_SQE_ALG_BITS 5
35 #define HPRE_SQE_DONE_SHIFT 30
36 #define HPRE_DH_MAX_P_SZ 512
37
38 #define HPRE_DFX_SEC_TO_US 1000000
39 #define HPRE_DFX_US_TO_NS 1000
40
41 #define HPRE_ENABLE_HPCORE_SHIFT 7
42
43 /* due to nist p521 */
44 #define HPRE_ECC_MAX_KSZ 66
45
46 /* size in bytes of the n prime */
47 #define HPRE_ECC_NIST_P192_N_SIZE 24
48 #define HPRE_ECC_NIST_P256_N_SIZE 32
49 #define HPRE_ECC_NIST_P384_N_SIZE 48
50
51 /* size in bytes */
52 #define HPRE_ECC_HW256_KSZ_B 32
53 #define HPRE_ECC_HW384_KSZ_B 48
54
55 /* capability register mask of driver */
56 #define HPRE_DRV_RSA_MASK_CAP BIT(0)
57 #define HPRE_DRV_DH_MASK_CAP BIT(1)
58 #define HPRE_DRV_ECDH_MASK_CAP BIT(2)
59 #define HPRE_DRV_X25519_MASK_CAP BIT(5)
60
61 static DEFINE_MUTEX(hpre_algs_lock);
62 static unsigned int hpre_available_devs;
63
64 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
65
66 struct hpre_rsa_ctx {
67 /* low address: e--->n */
68 char *pubkey;
69 dma_addr_t dma_pubkey;
70
71 /* low address: d--->n */
72 char *prikey;
73 dma_addr_t dma_prikey;
74
75 /* low address: dq->dp->q->p->qinv */
76 char *crt_prikey;
77 dma_addr_t dma_crt_prikey;
78
79 struct crypto_akcipher *soft_tfm;
80 };
81
82 struct hpre_dh_ctx {
83 /*
84 * If base is g we compute the public key
85 * ya = g^xa mod p; [RFC2631 sec 2.1.1]
86 * else if base if the counterpart public key we
87 * compute the shared secret
88 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
89 * low address: d--->n, please refer to Hisilicon HPRE UM
90 */
91 char *xa_p;
92 dma_addr_t dma_xa_p;
93
94 char *g; /* m */
95 dma_addr_t dma_g;
96 };
97
98 struct hpre_ecdh_ctx {
99 /* low address: p->a->k->b */
100 unsigned char *p;
101 dma_addr_t dma_p;
102
103 /* low address: x->y */
104 unsigned char *g;
105 dma_addr_t dma_g;
106 };
107
108 struct hpre_ctx {
109 struct hisi_qp *qp;
110 struct device *dev;
111 struct hpre_asym_request **req_list;
112 struct hpre *hpre;
113 spinlock_t req_lock;
114 unsigned int key_sz;
115 bool crt_g2_mode;
116 struct idr req_idr;
117 union {
118 struct hpre_rsa_ctx rsa;
119 struct hpre_dh_ctx dh;
120 struct hpre_ecdh_ctx ecdh;
121 };
122 /* for ecc algorithms */
123 unsigned int curve_id;
124 /* for high performance core */
125 u8 enable_hpcore;
126 };
127
128 struct hpre_asym_request {
129 char *src;
130 char *dst;
131 struct hpre_sqe req;
132 struct hpre_ctx *ctx;
133 union {
134 struct akcipher_request *rsa;
135 struct kpp_request *dh;
136 struct kpp_request *ecdh;
137 } areq;
138 int err;
139 int req_id;
140 hpre_cb cb;
141 struct timespec64 req_time;
142 };
143
hpre_align_sz(void)144 static inline unsigned int hpre_align_sz(void)
145 {
146 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;
147 }
148
hpre_align_pd(void)149 static inline unsigned int hpre_align_pd(void)
150 {
151 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);
152 }
153
hpre_alloc_req_id(struct hpre_ctx * ctx)154 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
155 {
156 unsigned long flags;
157 int id;
158
159 spin_lock_irqsave(&ctx->req_lock, flags);
160 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC);
161 spin_unlock_irqrestore(&ctx->req_lock, flags);
162
163 return id;
164 }
165
hpre_free_req_id(struct hpre_ctx * ctx,int req_id)166 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
167 {
168 unsigned long flags;
169
170 spin_lock_irqsave(&ctx->req_lock, flags);
171 idr_remove(&ctx->req_idr, req_id);
172 spin_unlock_irqrestore(&ctx->req_lock, flags);
173 }
174
hpre_add_req_to_ctx(struct hpre_asym_request * hpre_req)175 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
176 {
177 struct hpre_ctx *ctx;
178 struct hpre_dfx *dfx;
179 int id;
180
181 ctx = hpre_req->ctx;
182 id = hpre_alloc_req_id(ctx);
183 if (unlikely(id < 0))
184 return -EINVAL;
185
186 ctx->req_list[id] = hpre_req;
187 hpre_req->req_id = id;
188
189 dfx = ctx->hpre->debug.dfx;
190 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
191 ktime_get_ts64(&hpre_req->req_time);
192
193 return id;
194 }
195
hpre_rm_req_from_ctx(struct hpre_asym_request * hpre_req)196 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
197 {
198 struct hpre_ctx *ctx = hpre_req->ctx;
199 int id = hpre_req->req_id;
200
201 if (hpre_req->req_id >= 0) {
202 hpre_req->req_id = HPRE_INVLD_REQ_ID;
203 ctx->req_list[id] = NULL;
204 hpre_free_req_id(ctx, id);
205 }
206 }
207
hpre_get_qp_and_start(u8 type)208 static struct hisi_qp *hpre_get_qp_and_start(u8 type)
209 {
210 struct hisi_qp *qp;
211 int ret;
212
213 qp = hpre_create_qp(type);
214 if (!qp) {
215 pr_err("Can not create hpre qp!\n");
216 return ERR_PTR(-ENODEV);
217 }
218
219 ret = hisi_qm_start_qp(qp, 0);
220 if (ret < 0) {
221 hisi_qm_free_qps(&qp, 1);
222 pci_err(qp->qm->pdev, "Can not start qp!\n");
223 return ERR_PTR(-EINVAL);
224 }
225
226 return qp;
227 }
228
hpre_get_data_dma_addr(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)229 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
230 struct scatterlist *data, unsigned int len,
231 int is_src, dma_addr_t *tmp)
232 {
233 struct device *dev = hpre_req->ctx->dev;
234 enum dma_data_direction dma_dir;
235
236 if (is_src) {
237 hpre_req->src = NULL;
238 dma_dir = DMA_TO_DEVICE;
239 } else {
240 hpre_req->dst = NULL;
241 dma_dir = DMA_FROM_DEVICE;
242 }
243 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
244 if (unlikely(dma_mapping_error(dev, *tmp))) {
245 dev_err(dev, "dma map data err!\n");
246 return -ENOMEM;
247 }
248
249 return 0;
250 }
251
hpre_prepare_dma_buf(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)252 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
253 struct scatterlist *data, unsigned int len,
254 int is_src, dma_addr_t *tmp)
255 {
256 struct hpre_ctx *ctx = hpre_req->ctx;
257 struct device *dev = ctx->dev;
258 void *ptr;
259 int shift;
260
261 shift = ctx->key_sz - len;
262 if (unlikely(shift < 0))
263 return -EINVAL;
264
265 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
266 if (unlikely(!ptr))
267 return -ENOMEM;
268
269 if (is_src) {
270 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
271 hpre_req->src = ptr;
272 } else {
273 hpre_req->dst = ptr;
274 }
275
276 return 0;
277 }
278
hpre_hw_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,int is_dh)279 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
280 struct scatterlist *data, unsigned int len,
281 int is_src, int is_dh)
282 {
283 struct hpre_sqe *msg = &hpre_req->req;
284 struct hpre_ctx *ctx = hpre_req->ctx;
285 dma_addr_t tmp = 0;
286 int ret;
287
288 /* when the data is dh's source, we should format it */
289 if ((sg_is_last(data) && len == ctx->key_sz) &&
290 ((is_dh && !is_src) || !is_dh))
291 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
292 else
293 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
294
295 if (unlikely(ret))
296 return ret;
297
298 if (is_src)
299 msg->in = cpu_to_le64(tmp);
300 else
301 msg->out = cpu_to_le64(tmp);
302
303 return 0;
304 }
305
hpre_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)306 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
307 struct hpre_asym_request *req,
308 struct scatterlist *dst,
309 struct scatterlist *src)
310 {
311 struct device *dev = ctx->dev;
312 struct hpre_sqe *sqe = &req->req;
313 dma_addr_t tmp;
314
315 tmp = le64_to_cpu(sqe->in);
316 if (unlikely(dma_mapping_error(dev, tmp)))
317 return;
318
319 if (src) {
320 if (req->src)
321 dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
322 else
323 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
324 }
325
326 tmp = le64_to_cpu(sqe->out);
327 if (unlikely(dma_mapping_error(dev, tmp)))
328 return;
329
330 if (req->dst) {
331 if (dst)
332 scatterwalk_map_and_copy(req->dst, dst, 0,
333 ctx->key_sz, 1);
334 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
335 } else {
336 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
337 }
338 }
339
hpre_alg_res_post_hf(struct hpre_ctx * ctx,struct hpre_sqe * sqe,void ** kreq)340 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
341 void **kreq)
342 {
343 struct hpre_asym_request *req;
344 unsigned int err, done, alg;
345 int id;
346
347 #define HPRE_NO_HW_ERR 0
348 #define HPRE_HW_TASK_DONE 3
349 #define HREE_HW_ERR_MASK GENMASK(10, 0)
350 #define HREE_SQE_DONE_MASK GENMASK(1, 0)
351 #define HREE_ALG_TYPE_MASK GENMASK(4, 0)
352 id = (int)le16_to_cpu(sqe->tag);
353 req = ctx->req_list[id];
354 hpre_rm_req_from_ctx(req);
355 *kreq = req;
356
357 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
358 HREE_HW_ERR_MASK;
359
360 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
361 HREE_SQE_DONE_MASK;
362
363 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
364 return 0;
365
366 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
367 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
368 alg, done, err);
369
370 return -EINVAL;
371 }
372
hpre_ctx_set(struct hpre_ctx * ctx,struct hisi_qp * qp,int qlen)373 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
374 {
375 struct hpre *hpre;
376
377 if (!ctx || !qp || qlen < 0)
378 return -EINVAL;
379
380 spin_lock_init(&ctx->req_lock);
381 ctx->qp = qp;
382 ctx->dev = &qp->qm->pdev->dev;
383
384 hpre = container_of(ctx->qp->qm, struct hpre, qm);
385 ctx->hpre = hpre;
386 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
387 if (!ctx->req_list)
388 return -ENOMEM;
389 ctx->key_sz = 0;
390 ctx->crt_g2_mode = false;
391 idr_init(&ctx->req_idr);
392
393 return 0;
394 }
395
hpre_ctx_clear(struct hpre_ctx * ctx,bool is_clear_all)396 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
397 {
398 if (is_clear_all) {
399 idr_destroy(&ctx->req_idr);
400 kfree(ctx->req_list);
401 hisi_qm_free_qps(&ctx->qp, 1);
402 }
403
404 ctx->crt_g2_mode = false;
405 ctx->key_sz = 0;
406 }
407
hpre_is_bd_timeout(struct hpre_asym_request * req,u64 overtime_thrhld)408 static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
409 u64 overtime_thrhld)
410 {
411 struct timespec64 reply_time;
412 u64 time_use_us;
413
414 ktime_get_ts64(&reply_time);
415 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
416 HPRE_DFX_SEC_TO_US +
417 (reply_time.tv_nsec - req->req_time.tv_nsec) /
418 HPRE_DFX_US_TO_NS;
419
420 if (time_use_us <= overtime_thrhld)
421 return false;
422
423 return true;
424 }
425
hpre_dh_cb(struct hpre_ctx * ctx,void * resp)426 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
427 {
428 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
429 struct hpre_asym_request *req;
430 struct kpp_request *areq;
431 u64 overtime_thrhld;
432 int ret;
433
434 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
435 areq = req->areq.dh;
436 areq->dst_len = ctx->key_sz;
437
438 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
439 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
440 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
441
442 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
443 kpp_request_complete(areq, ret);
444 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
445 }
446
hpre_rsa_cb(struct hpre_ctx * ctx,void * resp)447 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
448 {
449 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
450 struct hpre_asym_request *req;
451 struct akcipher_request *areq;
452 u64 overtime_thrhld;
453 int ret;
454
455 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
456
457 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
458 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
459 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
460
461 areq = req->areq.rsa;
462 areq->dst_len = ctx->key_sz;
463 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
464 akcipher_request_complete(areq, ret);
465 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
466 }
467
hpre_alg_cb(struct hisi_qp * qp,void * resp)468 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
469 {
470 struct hpre_ctx *ctx = qp->qp_ctx;
471 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
472 struct hpre_sqe *sqe = resp;
473 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
474
475 if (unlikely(!req)) {
476 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
477 return;
478 }
479
480 req->cb(ctx, resp);
481 }
482
hpre_stop_qp_and_put(struct hisi_qp * qp)483 static void hpre_stop_qp_and_put(struct hisi_qp *qp)
484 {
485 hisi_qm_stop_qp(qp);
486 hisi_qm_free_qps(&qp, 1);
487 }
488
hpre_ctx_init(struct hpre_ctx * ctx,u8 type)489 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
490 {
491 struct hisi_qp *qp;
492 int ret;
493
494 qp = hpre_get_qp_and_start(type);
495 if (IS_ERR(qp))
496 return PTR_ERR(qp);
497
498 qp->qp_ctx = ctx;
499 qp->req_cb = hpre_alg_cb;
500
501 ret = hpre_ctx_set(ctx, qp, qp->sq_depth);
502 if (ret)
503 hpre_stop_qp_and_put(qp);
504
505 return ret;
506 }
507
hpre_msg_request_set(struct hpre_ctx * ctx,void * req,bool is_rsa)508 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
509 {
510 struct hpre_asym_request *h_req;
511 struct hpre_sqe *msg;
512 int req_id;
513 void *tmp;
514
515 if (is_rsa) {
516 struct akcipher_request *akreq = req;
517
518 if (akreq->dst_len < ctx->key_sz) {
519 akreq->dst_len = ctx->key_sz;
520 return -EOVERFLOW;
521 }
522
523 tmp = akcipher_request_ctx(akreq);
524 h_req = PTR_ALIGN(tmp, hpre_align_sz());
525 h_req->cb = hpre_rsa_cb;
526 h_req->areq.rsa = akreq;
527 msg = &h_req->req;
528 memset(msg, 0, sizeof(*msg));
529 } else {
530 struct kpp_request *kreq = req;
531
532 if (kreq->dst_len < ctx->key_sz) {
533 kreq->dst_len = ctx->key_sz;
534 return -EOVERFLOW;
535 }
536
537 tmp = kpp_request_ctx(kreq);
538 h_req = PTR_ALIGN(tmp, hpre_align_sz());
539 h_req->cb = hpre_dh_cb;
540 h_req->areq.dh = kreq;
541 msg = &h_req->req;
542 memset(msg, 0, sizeof(*msg));
543 msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
544 }
545
546 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
547 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
548 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
549 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
550 h_req->ctx = ctx;
551
552 req_id = hpre_add_req_to_ctx(h_req);
553 if (req_id < 0)
554 return -EBUSY;
555
556 msg->tag = cpu_to_le16((u16)req_id);
557
558 return 0;
559 }
560
hpre_send(struct hpre_ctx * ctx,struct hpre_sqe * msg)561 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
562 {
563 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
564 int ctr = 0;
565 int ret;
566
567 do {
568 atomic64_inc(&dfx[HPRE_SEND_CNT].value);
569 spin_lock_bh(&ctx->req_lock);
570 ret = hisi_qp_send(ctx->qp, msg);
571 spin_unlock_bh(&ctx->req_lock);
572 if (ret != -EBUSY)
573 break;
574 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
575 } while (ctr++ < HPRE_TRY_SEND_TIMES);
576
577 if (likely(!ret))
578 return ret;
579
580 if (ret != -EBUSY)
581 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
582
583 return ret;
584 }
585
hpre_dh_compute_value(struct kpp_request * req)586 static int hpre_dh_compute_value(struct kpp_request *req)
587 {
588 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
589 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
590 void *tmp = kpp_request_ctx(req);
591 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
592 struct hpre_sqe *msg = &hpre_req->req;
593 int ret;
594
595 ret = hpre_msg_request_set(ctx, req, false);
596 if (unlikely(ret))
597 return ret;
598
599 if (req->src) {
600 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
601 if (unlikely(ret))
602 goto clear_all;
603 } else {
604 msg->in = cpu_to_le64(ctx->dh.dma_g);
605 }
606
607 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
608 if (unlikely(ret))
609 goto clear_all;
610
611 if (ctx->crt_g2_mode && !req->src)
612 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
613 else
614 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
615
616 /* success */
617 ret = hpre_send(ctx, msg);
618 if (likely(!ret))
619 return -EINPROGRESS;
620
621 clear_all:
622 hpre_rm_req_from_ctx(hpre_req);
623 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
624
625 return ret;
626 }
627
hpre_is_dh_params_length_valid(unsigned int key_sz)628 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
629 {
630 #define _HPRE_DH_GRP1 768
631 #define _HPRE_DH_GRP2 1024
632 #define _HPRE_DH_GRP5 1536
633 #define _HPRE_DH_GRP14 2048
634 #define _HPRE_DH_GRP15 3072
635 #define _HPRE_DH_GRP16 4096
636 switch (key_sz) {
637 case _HPRE_DH_GRP1:
638 case _HPRE_DH_GRP2:
639 case _HPRE_DH_GRP5:
640 case _HPRE_DH_GRP14:
641 case _HPRE_DH_GRP15:
642 case _HPRE_DH_GRP16:
643 return 0;
644 default:
645 return -EINVAL;
646 }
647 }
648
hpre_dh_set_params(struct hpre_ctx * ctx,struct dh * params)649 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
650 {
651 struct device *dev = ctx->dev;
652 unsigned int sz;
653
654 if (params->p_size > HPRE_DH_MAX_P_SZ)
655 return -EINVAL;
656
657 if (hpre_is_dh_params_length_valid(params->p_size <<
658 HPRE_BITS_2_BYTES_SHIFT))
659 return -EINVAL;
660
661 sz = ctx->key_sz = params->p_size;
662 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
663 &ctx->dh.dma_xa_p, GFP_KERNEL);
664 if (!ctx->dh.xa_p)
665 return -ENOMEM;
666
667 memcpy(ctx->dh.xa_p + sz, params->p, sz);
668
669 /* If g equals 2 don't copy it */
670 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
671 ctx->crt_g2_mode = true;
672 return 0;
673 }
674
675 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
676 if (!ctx->dh.g) {
677 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
678 ctx->dh.dma_xa_p);
679 ctx->dh.xa_p = NULL;
680 return -ENOMEM;
681 }
682
683 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
684
685 return 0;
686 }
687
hpre_dh_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)688 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
689 {
690 struct device *dev = ctx->dev;
691 unsigned int sz = ctx->key_sz;
692
693 if (is_clear_all)
694 hisi_qm_stop_qp(ctx->qp);
695
696 if (ctx->dh.g) {
697 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
698 ctx->dh.g = NULL;
699 }
700
701 if (ctx->dh.xa_p) {
702 memzero_explicit(ctx->dh.xa_p, sz);
703 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
704 ctx->dh.dma_xa_p);
705 ctx->dh.xa_p = NULL;
706 }
707
708 hpre_ctx_clear(ctx, is_clear_all);
709 }
710
hpre_dh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)711 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
712 unsigned int len)
713 {
714 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
715 struct dh params;
716 int ret;
717
718 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)
719 return -EINVAL;
720
721 /* Free old secret if any */
722 hpre_dh_clear_ctx(ctx, false);
723
724 ret = hpre_dh_set_params(ctx, ¶ms);
725 if (ret < 0)
726 goto err_clear_ctx;
727
728 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
729 params.key_size);
730
731 return 0;
732
733 err_clear_ctx:
734 hpre_dh_clear_ctx(ctx, false);
735 return ret;
736 }
737
hpre_dh_max_size(struct crypto_kpp * tfm)738 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
739 {
740 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
741
742 return ctx->key_sz;
743 }
744
hpre_dh_init_tfm(struct crypto_kpp * tfm)745 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
746 {
747 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
748
749 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
750
751 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
752 }
753
hpre_dh_exit_tfm(struct crypto_kpp * tfm)754 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
755 {
756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
757
758 hpre_dh_clear_ctx(ctx, true);
759 }
760
hpre_rsa_drop_leading_zeros(const char ** ptr,size_t * len)761 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
762 {
763 while (!**ptr && *len) {
764 (*ptr)++;
765 (*len)--;
766 }
767 }
768
hpre_rsa_key_size_is_support(unsigned int len)769 static bool hpre_rsa_key_size_is_support(unsigned int len)
770 {
771 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
772
773 #define _RSA_1024BITS_KEY_WDTH 1024
774 #define _RSA_2048BITS_KEY_WDTH 2048
775 #define _RSA_3072BITS_KEY_WDTH 3072
776 #define _RSA_4096BITS_KEY_WDTH 4096
777
778 switch (bits) {
779 case _RSA_1024BITS_KEY_WDTH:
780 case _RSA_2048BITS_KEY_WDTH:
781 case _RSA_3072BITS_KEY_WDTH:
782 case _RSA_4096BITS_KEY_WDTH:
783 return true;
784 default:
785 return false;
786 }
787 }
788
hpre_rsa_enc(struct akcipher_request * req)789 static int hpre_rsa_enc(struct akcipher_request *req)
790 {
791 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
792 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
793 void *tmp = akcipher_request_ctx(req);
794 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
795 struct hpre_sqe *msg = &hpre_req->req;
796 int ret;
797
798 /* For 512 and 1536 bits key size, use soft tfm instead */
799 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
800 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
801 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
802 ret = crypto_akcipher_encrypt(req);
803 akcipher_request_set_tfm(req, tfm);
804 return ret;
805 }
806
807 if (unlikely(!ctx->rsa.pubkey))
808 return -EINVAL;
809
810 ret = hpre_msg_request_set(ctx, req, true);
811 if (unlikely(ret))
812 return ret;
813
814 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
815 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
816
817 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
818 if (unlikely(ret))
819 goto clear_all;
820
821 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
822 if (unlikely(ret))
823 goto clear_all;
824
825 /* success */
826 ret = hpre_send(ctx, msg);
827 if (likely(!ret))
828 return -EINPROGRESS;
829
830 clear_all:
831 hpre_rm_req_from_ctx(hpre_req);
832 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
833
834 return ret;
835 }
836
hpre_rsa_dec(struct akcipher_request * req)837 static int hpre_rsa_dec(struct akcipher_request *req)
838 {
839 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
840 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
841 void *tmp = akcipher_request_ctx(req);
842 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
843 struct hpre_sqe *msg = &hpre_req->req;
844 int ret;
845
846 /* For 512 and 1536 bits key size, use soft tfm instead */
847 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
848 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
849 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
850 ret = crypto_akcipher_decrypt(req);
851 akcipher_request_set_tfm(req, tfm);
852 return ret;
853 }
854
855 if (unlikely(!ctx->rsa.prikey))
856 return -EINVAL;
857
858 ret = hpre_msg_request_set(ctx, req, true);
859 if (unlikely(ret))
860 return ret;
861
862 if (ctx->crt_g2_mode) {
863 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
864 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
865 HPRE_ALG_NC_CRT);
866 } else {
867 msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
868 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
869 HPRE_ALG_NC_NCRT);
870 }
871
872 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
873 if (unlikely(ret))
874 goto clear_all;
875
876 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
877 if (unlikely(ret))
878 goto clear_all;
879
880 /* success */
881 ret = hpre_send(ctx, msg);
882 if (likely(!ret))
883 return -EINPROGRESS;
884
885 clear_all:
886 hpre_rm_req_from_ctx(hpre_req);
887 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
888
889 return ret;
890 }
891
hpre_rsa_set_n(struct hpre_ctx * ctx,const char * value,size_t vlen,bool private)892 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
893 size_t vlen, bool private)
894 {
895 const char *ptr = value;
896
897 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
898
899 ctx->key_sz = vlen;
900
901 /* if invalid key size provided, we use software tfm */
902 if (!hpre_rsa_key_size_is_support(ctx->key_sz))
903 return 0;
904
905 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
906 &ctx->rsa.dma_pubkey,
907 GFP_KERNEL);
908 if (!ctx->rsa.pubkey)
909 return -ENOMEM;
910
911 if (private) {
912 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
913 &ctx->rsa.dma_prikey,
914 GFP_KERNEL);
915 if (!ctx->rsa.prikey) {
916 dma_free_coherent(ctx->dev, vlen << 1,
917 ctx->rsa.pubkey,
918 ctx->rsa.dma_pubkey);
919 ctx->rsa.pubkey = NULL;
920 return -ENOMEM;
921 }
922 memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
923 }
924 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
925
926 /* Using hardware HPRE to do RSA */
927 return 1;
928 }
929
hpre_rsa_set_e(struct hpre_ctx * ctx,const char * value,size_t vlen)930 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
931 size_t vlen)
932 {
933 const char *ptr = value;
934
935 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
936
937 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
938 return -EINVAL;
939
940 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
941
942 return 0;
943 }
944
hpre_rsa_set_d(struct hpre_ctx * ctx,const char * value,size_t vlen)945 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
946 size_t vlen)
947 {
948 const char *ptr = value;
949
950 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
951
952 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
953 return -EINVAL;
954
955 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
956
957 return 0;
958 }
959
hpre_crt_para_get(char * para,size_t para_sz,const char * raw,size_t raw_sz)960 static int hpre_crt_para_get(char *para, size_t para_sz,
961 const char *raw, size_t raw_sz)
962 {
963 const char *ptr = raw;
964 size_t len = raw_sz;
965
966 hpre_rsa_drop_leading_zeros(&ptr, &len);
967 if (!len || len > para_sz)
968 return -EINVAL;
969
970 memcpy(para + para_sz - len, ptr, len);
971
972 return 0;
973 }
974
hpre_rsa_setkey_crt(struct hpre_ctx * ctx,struct rsa_key * rsa_key)975 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
976 {
977 unsigned int hlf_ksz = ctx->key_sz >> 1;
978 struct device *dev = ctx->dev;
979 u64 offset;
980 int ret;
981
982 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
983 &ctx->rsa.dma_crt_prikey,
984 GFP_KERNEL);
985 if (!ctx->rsa.crt_prikey)
986 return -ENOMEM;
987
988 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
989 rsa_key->dq, rsa_key->dq_sz);
990 if (ret)
991 goto free_key;
992
993 offset = hlf_ksz;
994 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
995 rsa_key->dp, rsa_key->dp_sz);
996 if (ret)
997 goto free_key;
998
999 offset = hlf_ksz * HPRE_CRT_Q;
1000 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1001 rsa_key->q, rsa_key->q_sz);
1002 if (ret)
1003 goto free_key;
1004
1005 offset = hlf_ksz * HPRE_CRT_P;
1006 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1007 rsa_key->p, rsa_key->p_sz);
1008 if (ret)
1009 goto free_key;
1010
1011 offset = hlf_ksz * HPRE_CRT_INV;
1012 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1013 rsa_key->qinv, rsa_key->qinv_sz);
1014 if (ret)
1015 goto free_key;
1016
1017 ctx->crt_g2_mode = true;
1018
1019 return 0;
1020
1021 free_key:
1022 offset = hlf_ksz * HPRE_CRT_PRMS;
1023 memzero_explicit(ctx->rsa.crt_prikey, offset);
1024 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1025 ctx->rsa.dma_crt_prikey);
1026 ctx->rsa.crt_prikey = NULL;
1027 ctx->crt_g2_mode = false;
1028
1029 return ret;
1030 }
1031
1032 /* If it is clear all, all the resources of the QP will be cleaned. */
hpre_rsa_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)1033 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1034 {
1035 unsigned int half_key_sz = ctx->key_sz >> 1;
1036 struct device *dev = ctx->dev;
1037
1038 if (is_clear_all)
1039 hisi_qm_stop_qp(ctx->qp);
1040
1041 if (ctx->rsa.pubkey) {
1042 dma_free_coherent(dev, ctx->key_sz << 1,
1043 ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1044 ctx->rsa.pubkey = NULL;
1045 }
1046
1047 if (ctx->rsa.crt_prikey) {
1048 memzero_explicit(ctx->rsa.crt_prikey,
1049 half_key_sz * HPRE_CRT_PRMS);
1050 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1051 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1052 ctx->rsa.crt_prikey = NULL;
1053 }
1054
1055 if (ctx->rsa.prikey) {
1056 memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1057 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1058 ctx->rsa.dma_prikey);
1059 ctx->rsa.prikey = NULL;
1060 }
1061
1062 hpre_ctx_clear(ctx, is_clear_all);
1063 }
1064
1065 /*
1066 * we should judge if it is CRT or not,
1067 * CRT: return true, N-CRT: return false .
1068 */
hpre_is_crt_key(struct rsa_key * key)1069 static bool hpre_is_crt_key(struct rsa_key *key)
1070 {
1071 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1072 key->qinv_sz;
1073
1074 #define LEN_OF_NCRT_PARA 5
1075
1076 /* N-CRT less than 5 parameters */
1077 return len > LEN_OF_NCRT_PARA;
1078 }
1079
hpre_rsa_setkey(struct hpre_ctx * ctx,const void * key,unsigned int keylen,bool private)1080 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1081 unsigned int keylen, bool private)
1082 {
1083 struct rsa_key rsa_key;
1084 int ret;
1085
1086 hpre_rsa_clear_ctx(ctx, false);
1087
1088 if (private)
1089 ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1090 else
1091 ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1092 if (ret < 0)
1093 return ret;
1094
1095 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1096 if (ret <= 0)
1097 return ret;
1098
1099 if (private) {
1100 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1101 if (ret < 0)
1102 goto free;
1103
1104 if (hpre_is_crt_key(&rsa_key)) {
1105 ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1106 if (ret < 0)
1107 goto free;
1108 }
1109 }
1110
1111 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1112 if (ret < 0)
1113 goto free;
1114
1115 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1116 ret = -EINVAL;
1117 goto free;
1118 }
1119
1120 return 0;
1121
1122 free:
1123 hpre_rsa_clear_ctx(ctx, false);
1124 return ret;
1125 }
1126
hpre_rsa_setpubkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1127 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1128 unsigned int keylen)
1129 {
1130 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1131 int ret;
1132
1133 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1134 if (ret)
1135 return ret;
1136
1137 return hpre_rsa_setkey(ctx, key, keylen, false);
1138 }
1139
hpre_rsa_setprivkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1140 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1141 unsigned int keylen)
1142 {
1143 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1144 int ret;
1145
1146 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1147 if (ret)
1148 return ret;
1149
1150 return hpre_rsa_setkey(ctx, key, keylen, true);
1151 }
1152
hpre_rsa_max_size(struct crypto_akcipher * tfm)1153 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1154 {
1155 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1156
1157 /* For 512 and 1536 bits key size, use soft tfm instead */
1158 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1159 ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1160 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1161
1162 return ctx->key_sz;
1163 }
1164
hpre_rsa_init_tfm(struct crypto_akcipher * tfm)1165 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1166 {
1167 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1168 int ret;
1169
1170 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1171 if (IS_ERR(ctx->rsa.soft_tfm)) {
1172 pr_err("Can not alloc_akcipher!\n");
1173 return PTR_ERR(ctx->rsa.soft_tfm);
1174 }
1175
1176 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +
1177 hpre_align_pd());
1178
1179 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1180 if (ret)
1181 crypto_free_akcipher(ctx->rsa.soft_tfm);
1182
1183 return ret;
1184 }
1185
hpre_rsa_exit_tfm(struct crypto_akcipher * tfm)1186 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1187 {
1188 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1189
1190 hpre_rsa_clear_ctx(ctx, true);
1191 crypto_free_akcipher(ctx->rsa.soft_tfm);
1192 }
1193
hpre_key_to_big_end(u8 * data,int len)1194 static void hpre_key_to_big_end(u8 *data, int len)
1195 {
1196 int i, j;
1197
1198 for (i = 0; i < len / 2; i++) {
1199 j = len - i - 1;
1200 swap(data[j], data[i]);
1201 }
1202 }
1203
hpre_ecc_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)1204 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1205 {
1206 struct device *dev = ctx->dev;
1207 unsigned int sz = ctx->key_sz;
1208 unsigned int shift = sz << 1;
1209
1210 if (is_clear_all)
1211 hisi_qm_stop_qp(ctx->qp);
1212
1213 if (ctx->ecdh.p) {
1214 /* ecdh: p->a->k->b */
1215 memzero_explicit(ctx->ecdh.p + shift, sz);
1216 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1217 ctx->ecdh.p = NULL;
1218 }
1219
1220 hpre_ctx_clear(ctx, is_clear_all);
1221 }
1222
1223 /*
1224 * The bits of 192/224/256/384/521 are supported by HPRE,
1225 * and convert the bits like:
1226 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
1227 * If the parameter bit width is insufficient, then we fill in the
1228 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
1229 */
hpre_ecdh_supported_curve(unsigned short id)1230 static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1231 {
1232 switch (id) {
1233 case ECC_CURVE_NIST_P192:
1234 case ECC_CURVE_NIST_P256:
1235 return HPRE_ECC_HW256_KSZ_B;
1236 case ECC_CURVE_NIST_P384:
1237 return HPRE_ECC_HW384_KSZ_B;
1238 default:
1239 break;
1240 }
1241
1242 return 0;
1243 }
1244
fill_curve_param(void * addr,u64 * param,unsigned int cur_sz,u8 ndigits)1245 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1246 {
1247 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1248 u8 i = 0;
1249
1250 while (i < ndigits - 1) {
1251 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64));
1252 i++;
1253 }
1254
1255 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz);
1256 hpre_key_to_big_end((u8 *)addr, cur_sz);
1257 }
1258
hpre_ecdh_fill_curve(struct hpre_ctx * ctx,struct ecdh * params,unsigned int cur_sz)1259 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1260 unsigned int cur_sz)
1261 {
1262 unsigned int shifta = ctx->key_sz << 1;
1263 unsigned int shiftb = ctx->key_sz << 2;
1264 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1265 void *a = ctx->ecdh.p + shifta - cur_sz;
1266 void *b = ctx->ecdh.p + shiftb - cur_sz;
1267 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1268 void *y = ctx->ecdh.g + shifta - cur_sz;
1269 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1270 char *n;
1271
1272 if (unlikely(!curve))
1273 return -EINVAL;
1274
1275 n = kzalloc(ctx->key_sz, GFP_KERNEL);
1276 if (!n)
1277 return -ENOMEM;
1278
1279 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1280 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1281 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1282 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1283 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1284 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1285
1286 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1287 kfree(n);
1288 return -EINVAL;
1289 }
1290
1291 kfree(n);
1292 return 0;
1293 }
1294
hpre_ecdh_get_curvesz(unsigned short id)1295 static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1296 {
1297 switch (id) {
1298 case ECC_CURVE_NIST_P192:
1299 return HPRE_ECC_NIST_P192_N_SIZE;
1300 case ECC_CURVE_NIST_P256:
1301 return HPRE_ECC_NIST_P256_N_SIZE;
1302 case ECC_CURVE_NIST_P384:
1303 return HPRE_ECC_NIST_P384_N_SIZE;
1304 default:
1305 break;
1306 }
1307
1308 return 0;
1309 }
1310
hpre_ecdh_set_param(struct hpre_ctx * ctx,struct ecdh * params)1311 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1312 {
1313 struct device *dev = ctx->dev;
1314 unsigned int sz, shift, curve_sz;
1315 int ret;
1316
1317 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1318 if (!ctx->key_sz)
1319 return -EINVAL;
1320
1321 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1322 if (!curve_sz || params->key_size > curve_sz)
1323 return -EINVAL;
1324
1325 sz = ctx->key_sz;
1326
1327 if (!ctx->ecdh.p) {
1328 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1329 GFP_KERNEL);
1330 if (!ctx->ecdh.p)
1331 return -ENOMEM;
1332 }
1333
1334 shift = sz << 2;
1335 ctx->ecdh.g = ctx->ecdh.p + shift;
1336 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1337
1338 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1339 if (ret) {
1340 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1341 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1342 ctx->ecdh.p = NULL;
1343 return ret;
1344 }
1345
1346 return 0;
1347 }
1348
hpre_key_is_zero(char * key,unsigned short key_sz)1349 static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1350 {
1351 int i;
1352
1353 for (i = 0; i < key_sz; i++)
1354 if (key[i])
1355 return false;
1356
1357 return true;
1358 }
1359
ecdh_gen_privkey(struct hpre_ctx * ctx,struct ecdh * params)1360 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
1361 {
1362 struct device *dev = ctx->dev;
1363 int ret;
1364
1365 ret = crypto_get_default_rng();
1366 if (ret) {
1367 dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
1368 return ret;
1369 }
1370
1371 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
1372 params->key_size);
1373 crypto_put_default_rng();
1374 if (ret)
1375 dev_err(dev, "failed to get rng, ret = %d!\n", ret);
1376
1377 return ret;
1378 }
1379
hpre_ecdh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1380 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1381 unsigned int len)
1382 {
1383 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1384 unsigned int sz, sz_shift, curve_sz;
1385 struct device *dev = ctx->dev;
1386 char key[HPRE_ECC_MAX_KSZ];
1387 struct ecdh params;
1388 int ret;
1389
1390 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) {
1391 dev_err(dev, "failed to decode ecdh key!\n");
1392 return -EINVAL;
1393 }
1394
1395 /* Use stdrng to generate private key */
1396 if (!params.key || !params.key_size) {
1397 params.key = key;
1398 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1399 if (!curve_sz) {
1400 dev_err(dev, "Invalid curve size!\n");
1401 return -EINVAL;
1402 }
1403
1404 params.key_size = curve_sz - 1;
1405 ret = ecdh_gen_privkey(ctx, ¶ms);
1406 if (ret)
1407 return ret;
1408 }
1409
1410 if (hpre_key_is_zero(params.key, params.key_size)) {
1411 dev_err(dev, "Invalid hpre key!\n");
1412 return -EINVAL;
1413 }
1414
1415 hpre_ecc_clear_ctx(ctx, false);
1416
1417 ret = hpre_ecdh_set_param(ctx, ¶ms);
1418 if (ret < 0) {
1419 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1420 return ret;
1421 }
1422
1423 sz = ctx->key_sz;
1424 sz_shift = (sz << 1) + sz - params.key_size;
1425 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1426
1427 return 0;
1428 }
1429
hpre_ecdh_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1430 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1431 struct hpre_asym_request *req,
1432 struct scatterlist *dst,
1433 struct scatterlist *src)
1434 {
1435 struct device *dev = ctx->dev;
1436 struct hpre_sqe *sqe = &req->req;
1437 dma_addr_t dma;
1438
1439 dma = le64_to_cpu(sqe->in);
1440 if (unlikely(dma_mapping_error(dev, dma)))
1441 return;
1442
1443 if (src && req->src)
1444 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1445
1446 dma = le64_to_cpu(sqe->out);
1447 if (unlikely(dma_mapping_error(dev, dma)))
1448 return;
1449
1450 if (req->dst)
1451 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1452 if (dst)
1453 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1454 }
1455
hpre_ecdh_cb(struct hpre_ctx * ctx,void * resp)1456 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1457 {
1458 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1459 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1460 struct hpre_asym_request *req = NULL;
1461 struct kpp_request *areq;
1462 u64 overtime_thrhld;
1463 char *p;
1464 int ret;
1465
1466 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1467 areq = req->areq.ecdh;
1468 areq->dst_len = ctx->key_sz << 1;
1469
1470 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1471 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1472 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1473
1474 /* Do unmap before data processing */
1475 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1476
1477 p = sg_virt(areq->dst);
1478 memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1479 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1480
1481 kpp_request_complete(areq, ret);
1482
1483 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1484 }
1485
hpre_ecdh_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1486 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1487 struct kpp_request *req)
1488 {
1489 struct hpre_asym_request *h_req;
1490 struct hpre_sqe *msg;
1491 int req_id;
1492 void *tmp;
1493
1494 if (req->dst_len < ctx->key_sz << 1) {
1495 req->dst_len = ctx->key_sz << 1;
1496 return -EINVAL;
1497 }
1498
1499 tmp = kpp_request_ctx(req);
1500 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1501 h_req->cb = hpre_ecdh_cb;
1502 h_req->areq.ecdh = req;
1503 msg = &h_req->req;
1504 memset(msg, 0, sizeof(*msg));
1505 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1506 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1507 msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1508
1509 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1510 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1511 h_req->ctx = ctx;
1512
1513 req_id = hpre_add_req_to_ctx(h_req);
1514 if (req_id < 0)
1515 return -EBUSY;
1516
1517 msg->tag = cpu_to_le16((u16)req_id);
1518 return 0;
1519 }
1520
hpre_ecdh_src_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1521 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1522 struct scatterlist *data, unsigned int len)
1523 {
1524 struct hpre_sqe *msg = &hpre_req->req;
1525 struct hpre_ctx *ctx = hpre_req->ctx;
1526 struct device *dev = ctx->dev;
1527 unsigned int tmpshift;
1528 dma_addr_t dma = 0;
1529 void *ptr;
1530 int shift;
1531
1532 /* Src_data include gx and gy. */
1533 shift = ctx->key_sz - (len >> 1);
1534 if (unlikely(shift < 0))
1535 return -EINVAL;
1536
1537 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1538 if (unlikely(!ptr))
1539 return -ENOMEM;
1540
1541 tmpshift = ctx->key_sz << 1;
1542 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1543 memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1544 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1545
1546 hpre_req->src = ptr;
1547 msg->in = cpu_to_le64(dma);
1548 return 0;
1549 }
1550
hpre_ecdh_dst_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1551 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1552 struct scatterlist *data, unsigned int len)
1553 {
1554 struct hpre_sqe *msg = &hpre_req->req;
1555 struct hpre_ctx *ctx = hpre_req->ctx;
1556 struct device *dev = ctx->dev;
1557 dma_addr_t dma;
1558
1559 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1560 dev_err(dev, "data or data length is illegal!\n");
1561 return -EINVAL;
1562 }
1563
1564 hpre_req->dst = NULL;
1565 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1566 if (unlikely(dma_mapping_error(dev, dma))) {
1567 dev_err(dev, "dma map data err!\n");
1568 return -ENOMEM;
1569 }
1570
1571 msg->out = cpu_to_le64(dma);
1572 return 0;
1573 }
1574
hpre_ecdh_compute_value(struct kpp_request * req)1575 static int hpre_ecdh_compute_value(struct kpp_request *req)
1576 {
1577 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1578 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1579 struct device *dev = ctx->dev;
1580 void *tmp = kpp_request_ctx(req);
1581 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1582 struct hpre_sqe *msg = &hpre_req->req;
1583 int ret;
1584
1585 ret = hpre_ecdh_msg_request_set(ctx, req);
1586 if (unlikely(ret)) {
1587 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1588 return ret;
1589 }
1590
1591 if (req->src) {
1592 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1593 if (unlikely(ret)) {
1594 dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1595 goto clear_all;
1596 }
1597 } else {
1598 msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1599 }
1600
1601 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1602 if (unlikely(ret)) {
1603 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1604 goto clear_all;
1605 }
1606
1607 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1608 msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT;
1609
1610 ret = hpre_send(ctx, msg);
1611 if (likely(!ret))
1612 return -EINPROGRESS;
1613
1614 clear_all:
1615 hpre_rm_req_from_ctx(hpre_req);
1616 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1617 return ret;
1618 }
1619
hpre_ecdh_max_size(struct crypto_kpp * tfm)1620 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1621 {
1622 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1623
1624 /* max size is the pub_key_size, include x and y */
1625 return ctx->key_sz << 1;
1626 }
1627
hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp * tfm)1628 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1629 {
1630 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1631
1632 ctx->curve_id = ECC_CURVE_NIST_P192;
1633
1634 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1635
1636 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1637 }
1638
hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp * tfm)1639 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1640 {
1641 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1642
1643 ctx->curve_id = ECC_CURVE_NIST_P256;
1644 ctx->enable_hpcore = 1;
1645
1646 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1647
1648 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1649 }
1650
hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp * tfm)1651 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
1652 {
1653 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1654
1655 ctx->curve_id = ECC_CURVE_NIST_P384;
1656
1657 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1658
1659 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1660 }
1661
hpre_ecdh_exit_tfm(struct crypto_kpp * tfm)1662 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1663 {
1664 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1665
1666 hpre_ecc_clear_ctx(ctx, true);
1667 }
1668
1669 static struct akcipher_alg rsa = {
1670 .encrypt = hpre_rsa_enc,
1671 .decrypt = hpre_rsa_dec,
1672 .set_pub_key = hpre_rsa_setpubkey,
1673 .set_priv_key = hpre_rsa_setprivkey,
1674 .max_size = hpre_rsa_max_size,
1675 .init = hpre_rsa_init_tfm,
1676 .exit = hpre_rsa_exit_tfm,
1677 .base = {
1678 .cra_ctxsize = sizeof(struct hpre_ctx),
1679 .cra_priority = HPRE_CRYPTO_ALG_PRI,
1680 .cra_name = "rsa",
1681 .cra_driver_name = "hpre-rsa",
1682 .cra_module = THIS_MODULE,
1683 },
1684 };
1685
1686 static struct kpp_alg dh = {
1687 .set_secret = hpre_dh_set_secret,
1688 .generate_public_key = hpre_dh_compute_value,
1689 .compute_shared_secret = hpre_dh_compute_value,
1690 .max_size = hpre_dh_max_size,
1691 .init = hpre_dh_init_tfm,
1692 .exit = hpre_dh_exit_tfm,
1693 .base = {
1694 .cra_ctxsize = sizeof(struct hpre_ctx),
1695 .cra_priority = HPRE_CRYPTO_ALG_PRI,
1696 .cra_name = "dh",
1697 .cra_driver_name = "hpre-dh",
1698 .cra_module = THIS_MODULE,
1699 },
1700 };
1701
1702 static struct kpp_alg ecdh_curves[] = {
1703 {
1704 .set_secret = hpre_ecdh_set_secret,
1705 .generate_public_key = hpre_ecdh_compute_value,
1706 .compute_shared_secret = hpre_ecdh_compute_value,
1707 .max_size = hpre_ecdh_max_size,
1708 .init = hpre_ecdh_nist_p192_init_tfm,
1709 .exit = hpre_ecdh_exit_tfm,
1710 .base = {
1711 .cra_ctxsize = sizeof(struct hpre_ctx),
1712 .cra_priority = HPRE_CRYPTO_ALG_PRI,
1713 .cra_name = "ecdh-nist-p192",
1714 .cra_driver_name = "hpre-ecdh-nist-p192",
1715 .cra_module = THIS_MODULE,
1716 },
1717 }, {
1718 .set_secret = hpre_ecdh_set_secret,
1719 .generate_public_key = hpre_ecdh_compute_value,
1720 .compute_shared_secret = hpre_ecdh_compute_value,
1721 .max_size = hpre_ecdh_max_size,
1722 .init = hpre_ecdh_nist_p256_init_tfm,
1723 .exit = hpre_ecdh_exit_tfm,
1724 .base = {
1725 .cra_ctxsize = sizeof(struct hpre_ctx),
1726 .cra_priority = HPRE_CRYPTO_ALG_PRI,
1727 .cra_name = "ecdh-nist-p256",
1728 .cra_driver_name = "hpre-ecdh-nist-p256",
1729 .cra_module = THIS_MODULE,
1730 },
1731 }, {
1732 .set_secret = hpre_ecdh_set_secret,
1733 .generate_public_key = hpre_ecdh_compute_value,
1734 .compute_shared_secret = hpre_ecdh_compute_value,
1735 .max_size = hpre_ecdh_max_size,
1736 .init = hpre_ecdh_nist_p384_init_tfm,
1737 .exit = hpre_ecdh_exit_tfm,
1738 .base = {
1739 .cra_ctxsize = sizeof(struct hpre_ctx),
1740 .cra_priority = HPRE_CRYPTO_ALG_PRI,
1741 .cra_name = "ecdh-nist-p384",
1742 .cra_driver_name = "hpre-ecdh-nist-p384",
1743 .cra_module = THIS_MODULE,
1744 },
1745 }
1746 };
1747
hpre_register_rsa(struct hisi_qm * qm)1748 static int hpre_register_rsa(struct hisi_qm *qm)
1749 {
1750 int ret;
1751
1752 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
1753 return 0;
1754
1755 rsa.base.cra_flags = 0;
1756 ret = crypto_register_akcipher(&rsa);
1757 if (ret)
1758 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);
1759
1760 return ret;
1761 }
1762
hpre_unregister_rsa(struct hisi_qm * qm)1763 static void hpre_unregister_rsa(struct hisi_qm *qm)
1764 {
1765 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
1766 return;
1767
1768 crypto_unregister_akcipher(&rsa);
1769 }
1770
hpre_register_dh(struct hisi_qm * qm)1771 static int hpre_register_dh(struct hisi_qm *qm)
1772 {
1773 int ret;
1774
1775 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
1776 return 0;
1777
1778 ret = crypto_register_kpp(&dh);
1779 if (ret)
1780 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);
1781
1782 return ret;
1783 }
1784
hpre_unregister_dh(struct hisi_qm * qm)1785 static void hpre_unregister_dh(struct hisi_qm *qm)
1786 {
1787 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
1788 return;
1789
1790 crypto_unregister_kpp(&dh);
1791 }
1792
hpre_register_ecdh(struct hisi_qm * qm)1793 static int hpre_register_ecdh(struct hisi_qm *qm)
1794 {
1795 int ret, i;
1796
1797 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
1798 return 0;
1799
1800 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {
1801 ret = crypto_register_kpp(&ecdh_curves[i]);
1802 if (ret) {
1803 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",
1804 ecdh_curves[i].base.cra_name, ret);
1805 goto unreg_kpp;
1806 }
1807 }
1808
1809 return 0;
1810
1811 unreg_kpp:
1812 for (--i; i >= 0; --i)
1813 crypto_unregister_kpp(&ecdh_curves[i]);
1814
1815 return ret;
1816 }
1817
hpre_unregister_ecdh(struct hisi_qm * qm)1818 static void hpre_unregister_ecdh(struct hisi_qm *qm)
1819 {
1820 int i;
1821
1822 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
1823 return;
1824
1825 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)
1826 crypto_unregister_kpp(&ecdh_curves[i]);
1827 }
1828
hpre_algs_register(struct hisi_qm * qm)1829 int hpre_algs_register(struct hisi_qm *qm)
1830 {
1831 int ret = 0;
1832
1833 mutex_lock(&hpre_algs_lock);
1834 if (hpre_available_devs) {
1835 hpre_available_devs++;
1836 goto unlock;
1837 }
1838
1839 ret = hpre_register_rsa(qm);
1840 if (ret)
1841 goto unlock;
1842
1843 ret = hpre_register_dh(qm);
1844 if (ret)
1845 goto unreg_rsa;
1846
1847 ret = hpre_register_ecdh(qm);
1848 if (ret)
1849 goto unreg_dh;
1850
1851 hpre_available_devs++;
1852 mutex_unlock(&hpre_algs_lock);
1853
1854 return ret;
1855
1856 unreg_dh:
1857 hpre_unregister_dh(qm);
1858 unreg_rsa:
1859 hpre_unregister_rsa(qm);
1860 unlock:
1861 mutex_unlock(&hpre_algs_lock);
1862 return ret;
1863 }
1864
hpre_algs_unregister(struct hisi_qm * qm)1865 void hpre_algs_unregister(struct hisi_qm *qm)
1866 {
1867 mutex_lock(&hpre_algs_lock);
1868 if (--hpre_available_devs)
1869 goto unlock;
1870
1871 hpre_unregister_ecdh(qm);
1872 hpre_unregister_dh(qm);
1873 hpre_unregister_rsa(qm);
1874
1875 unlock:
1876 mutex_unlock(&hpre_algs_lock);
1877 }
1878