1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/curve25519.h>
5 #include <crypto/dh.h>
6 #include <crypto/ecc_curve.h>
7 #include <crypto/ecdh.h>
8 #include <crypto/rng.h>
9 #include <crypto/internal/akcipher.h>
10 #include <crypto/internal/kpp.h>
11 #include <crypto/internal/rsa.h>
12 #include <crypto/kpp.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/fips.h>
16 #include <linux/module.h>
17 #include <linux/time.h>
18 #include "hpre.h"
19
20 struct hpre_ctx;
21
22 #define HPRE_CRYPTO_ALG_PRI 1000
23 #define HPRE_ALIGN_SZ 64
24 #define HPRE_BITS_2_BYTES_SHIFT 3
25 #define HPRE_RSA_512BITS_KSZ 64
26 #define HPRE_RSA_1536BITS_KSZ 192
27 #define HPRE_CRT_PRMS 5
28 #define HPRE_CRT_Q 2
29 #define HPRE_CRT_P 3
30 #define HPRE_CRT_INV 4
31 #define HPRE_DH_G_FLAG 0x02
32 #define HPRE_TRY_SEND_TIMES 100
33 #define HPRE_INVLD_REQ_ID (-1)
34
35 #define HPRE_SQE_ALG_BITS 5
36 #define HPRE_SQE_DONE_SHIFT 30
37 #define HPRE_DH_MAX_P_SZ 512
38
39 #define HPRE_DFX_SEC_TO_US 1000000
40 #define HPRE_DFX_US_TO_NS 1000
41
42 /* due to nist p521 */
43 #define HPRE_ECC_MAX_KSZ 66
44
45 /* size in bytes of the n prime */
46 #define HPRE_ECC_NIST_P192_N_SIZE 24
47 #define HPRE_ECC_NIST_P256_N_SIZE 32
48 #define HPRE_ECC_NIST_P384_N_SIZE 48
49
50 /* size in bytes */
51 #define HPRE_ECC_HW256_KSZ_B 32
52 #define HPRE_ECC_HW384_KSZ_B 48
53
54 /* capability register mask of driver */
55 #define HPRE_DRV_RSA_MASK_CAP BIT(0)
56 #define HPRE_DRV_DH_MASK_CAP BIT(1)
57 #define HPRE_DRV_ECDH_MASK_CAP BIT(2)
58 #define HPRE_DRV_X25519_MASK_CAP BIT(5)
59
60 static DEFINE_MUTEX(hpre_algs_lock);
61 static unsigned int hpre_available_devs;
62
63 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
64
65 struct hpre_rsa_ctx {
66 /* low address: e--->n */
67 char *pubkey;
68 dma_addr_t dma_pubkey;
69
70 /* low address: d--->n */
71 char *prikey;
72 dma_addr_t dma_prikey;
73
74 /* low address: dq->dp->q->p->qinv */
75 char *crt_prikey;
76 dma_addr_t dma_crt_prikey;
77
78 struct crypto_akcipher *soft_tfm;
79 };
80
81 struct hpre_dh_ctx {
82 /*
83 * If base is g we compute the public key
84 * ya = g^xa mod p; [RFC2631 sec 2.1.1]
85 * else if base if the counterpart public key we
86 * compute the shared secret
87 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
88 * low address: d--->n, please refer to Hisilicon HPRE UM
89 */
90 char *xa_p;
91 dma_addr_t dma_xa_p;
92
93 char *g; /* m */
94 dma_addr_t dma_g;
95 };
96
97 struct hpre_ecdh_ctx {
98 /* low address: p->a->k->b */
99 unsigned char *p;
100 dma_addr_t dma_p;
101
102 /* low address: x->y */
103 unsigned char *g;
104 dma_addr_t dma_g;
105 };
106
107 struct hpre_curve25519_ctx {
108 /* low address: p->a->k */
109 unsigned char *p;
110 dma_addr_t dma_p;
111
112 /* gx coordinate */
113 unsigned char *g;
114 dma_addr_t dma_g;
115 };
116
117 struct hpre_ctx {
118 struct hisi_qp *qp;
119 struct device *dev;
120 struct hpre_asym_request **req_list;
121 struct hpre *hpre;
122 spinlock_t req_lock;
123 unsigned int key_sz;
124 bool crt_g2_mode;
125 struct idr req_idr;
126 union {
127 struct hpre_rsa_ctx rsa;
128 struct hpre_dh_ctx dh;
129 struct hpre_ecdh_ctx ecdh;
130 struct hpre_curve25519_ctx curve25519;
131 };
132 /* for ecc algorithms */
133 unsigned int curve_id;
134 };
135
136 struct hpre_asym_request {
137 char *src;
138 char *dst;
139 struct hpre_sqe req;
140 struct hpre_ctx *ctx;
141 union {
142 struct akcipher_request *rsa;
143 struct kpp_request *dh;
144 struct kpp_request *ecdh;
145 struct kpp_request *curve25519;
146 } areq;
147 int err;
148 int req_id;
149 hpre_cb cb;
150 struct timespec64 req_time;
151 };
152
hpre_align_sz(void)153 static inline unsigned int hpre_align_sz(void)
154 {
155 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;
156 }
157
hpre_align_pd(void)158 static inline unsigned int hpre_align_pd(void)
159 {
160 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);
161 }
162
hpre_alloc_req_id(struct hpre_ctx * ctx)163 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
164 {
165 unsigned long flags;
166 int id;
167
168 spin_lock_irqsave(&ctx->req_lock, flags);
169 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC);
170 spin_unlock_irqrestore(&ctx->req_lock, flags);
171
172 return id;
173 }
174
hpre_free_req_id(struct hpre_ctx * ctx,int req_id)175 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
176 {
177 unsigned long flags;
178
179 spin_lock_irqsave(&ctx->req_lock, flags);
180 idr_remove(&ctx->req_idr, req_id);
181 spin_unlock_irqrestore(&ctx->req_lock, flags);
182 }
183
hpre_add_req_to_ctx(struct hpre_asym_request * hpre_req)184 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
185 {
186 struct hpre_ctx *ctx;
187 struct hpre_dfx *dfx;
188 int id;
189
190 ctx = hpre_req->ctx;
191 id = hpre_alloc_req_id(ctx);
192 if (unlikely(id < 0))
193 return -EINVAL;
194
195 ctx->req_list[id] = hpre_req;
196 hpre_req->req_id = id;
197
198 dfx = ctx->hpre->debug.dfx;
199 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
200 ktime_get_ts64(&hpre_req->req_time);
201
202 return id;
203 }
204
hpre_rm_req_from_ctx(struct hpre_asym_request * hpre_req)205 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
206 {
207 struct hpre_ctx *ctx = hpre_req->ctx;
208 int id = hpre_req->req_id;
209
210 if (hpre_req->req_id >= 0) {
211 hpre_req->req_id = HPRE_INVLD_REQ_ID;
212 ctx->req_list[id] = NULL;
213 hpre_free_req_id(ctx, id);
214 }
215 }
216
hpre_get_qp_and_start(u8 type)217 static struct hisi_qp *hpre_get_qp_and_start(u8 type)
218 {
219 struct hisi_qp *qp;
220 int ret;
221
222 qp = hpre_create_qp(type);
223 if (!qp) {
224 pr_err("Can not create hpre qp!\n");
225 return ERR_PTR(-ENODEV);
226 }
227
228 ret = hisi_qm_start_qp(qp, 0);
229 if (ret < 0) {
230 hisi_qm_free_qps(&qp, 1);
231 pci_err(qp->qm->pdev, "Can not start qp!\n");
232 return ERR_PTR(-EINVAL);
233 }
234
235 return qp;
236 }
237
hpre_get_data_dma_addr(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)238 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
239 struct scatterlist *data, unsigned int len,
240 int is_src, dma_addr_t *tmp)
241 {
242 struct device *dev = hpre_req->ctx->dev;
243 enum dma_data_direction dma_dir;
244
245 if (is_src) {
246 hpre_req->src = NULL;
247 dma_dir = DMA_TO_DEVICE;
248 } else {
249 hpre_req->dst = NULL;
250 dma_dir = DMA_FROM_DEVICE;
251 }
252 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
253 if (unlikely(dma_mapping_error(dev, *tmp))) {
254 dev_err(dev, "dma map data err!\n");
255 return -ENOMEM;
256 }
257
258 return 0;
259 }
260
hpre_prepare_dma_buf(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)261 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
262 struct scatterlist *data, unsigned int len,
263 int is_src, dma_addr_t *tmp)
264 {
265 struct hpre_ctx *ctx = hpre_req->ctx;
266 struct device *dev = ctx->dev;
267 void *ptr;
268 int shift;
269
270 shift = ctx->key_sz - len;
271 if (unlikely(shift < 0))
272 return -EINVAL;
273
274 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
275 if (unlikely(!ptr))
276 return -ENOMEM;
277
278 if (is_src) {
279 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
280 hpre_req->src = ptr;
281 } else {
282 hpre_req->dst = ptr;
283 }
284
285 return 0;
286 }
287
hpre_hw_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,int is_dh)288 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
289 struct scatterlist *data, unsigned int len,
290 int is_src, int is_dh)
291 {
292 struct hpre_sqe *msg = &hpre_req->req;
293 struct hpre_ctx *ctx = hpre_req->ctx;
294 dma_addr_t tmp = 0;
295 int ret;
296
297 /* when the data is dh's source, we should format it */
298 if ((sg_is_last(data) && len == ctx->key_sz) &&
299 ((is_dh && !is_src) || !is_dh))
300 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
301 else
302 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
303
304 if (unlikely(ret))
305 return ret;
306
307 if (is_src)
308 msg->in = cpu_to_le64(tmp);
309 else
310 msg->out = cpu_to_le64(tmp);
311
312 return 0;
313 }
314
hpre_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)315 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
316 struct hpre_asym_request *req,
317 struct scatterlist *dst,
318 struct scatterlist *src)
319 {
320 struct device *dev = ctx->dev;
321 struct hpre_sqe *sqe = &req->req;
322 dma_addr_t tmp;
323
324 tmp = le64_to_cpu(sqe->in);
325 if (unlikely(dma_mapping_error(dev, tmp)))
326 return;
327
328 if (src) {
329 if (req->src)
330 dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
331 else
332 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
333 }
334
335 tmp = le64_to_cpu(sqe->out);
336 if (unlikely(dma_mapping_error(dev, tmp)))
337 return;
338
339 if (req->dst) {
340 if (dst)
341 scatterwalk_map_and_copy(req->dst, dst, 0,
342 ctx->key_sz, 1);
343 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
344 } else {
345 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
346 }
347 }
348
hpre_alg_res_post_hf(struct hpre_ctx * ctx,struct hpre_sqe * sqe,void ** kreq)349 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
350 void **kreq)
351 {
352 struct hpre_asym_request *req;
353 unsigned int err, done, alg;
354 int id;
355
356 #define HPRE_NO_HW_ERR 0
357 #define HPRE_HW_TASK_DONE 3
358 #define HREE_HW_ERR_MASK GENMASK(10, 0)
359 #define HREE_SQE_DONE_MASK GENMASK(1, 0)
360 #define HREE_ALG_TYPE_MASK GENMASK(4, 0)
361 id = (int)le16_to_cpu(sqe->tag);
362 req = ctx->req_list[id];
363 hpre_rm_req_from_ctx(req);
364 *kreq = req;
365
366 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
367 HREE_HW_ERR_MASK;
368
369 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
370 HREE_SQE_DONE_MASK;
371
372 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
373 return 0;
374
375 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
376 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
377 alg, done, err);
378
379 return -EINVAL;
380 }
381
hpre_ctx_set(struct hpre_ctx * ctx,struct hisi_qp * qp,int qlen)382 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
383 {
384 struct hpre *hpre;
385
386 if (!ctx || !qp || qlen < 0)
387 return -EINVAL;
388
389 spin_lock_init(&ctx->req_lock);
390 ctx->qp = qp;
391 ctx->dev = &qp->qm->pdev->dev;
392
393 hpre = container_of(ctx->qp->qm, struct hpre, qm);
394 ctx->hpre = hpre;
395 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
396 if (!ctx->req_list)
397 return -ENOMEM;
398 ctx->key_sz = 0;
399 ctx->crt_g2_mode = false;
400 idr_init(&ctx->req_idr);
401
402 return 0;
403 }
404
hpre_ctx_clear(struct hpre_ctx * ctx,bool is_clear_all)405 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
406 {
407 if (is_clear_all) {
408 idr_destroy(&ctx->req_idr);
409 kfree(ctx->req_list);
410 hisi_qm_free_qps(&ctx->qp, 1);
411 }
412
413 ctx->crt_g2_mode = false;
414 ctx->key_sz = 0;
415 }
416
hpre_is_bd_timeout(struct hpre_asym_request * req,u64 overtime_thrhld)417 static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
418 u64 overtime_thrhld)
419 {
420 struct timespec64 reply_time;
421 u64 time_use_us;
422
423 ktime_get_ts64(&reply_time);
424 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
425 HPRE_DFX_SEC_TO_US +
426 (reply_time.tv_nsec - req->req_time.tv_nsec) /
427 HPRE_DFX_US_TO_NS;
428
429 if (time_use_us <= overtime_thrhld)
430 return false;
431
432 return true;
433 }
434
hpre_dh_cb(struct hpre_ctx * ctx,void * resp)435 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
436 {
437 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
438 struct hpre_asym_request *req;
439 struct kpp_request *areq;
440 u64 overtime_thrhld;
441 int ret;
442
443 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
444 areq = req->areq.dh;
445 areq->dst_len = ctx->key_sz;
446
447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
450
451 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
452 kpp_request_complete(areq, ret);
453 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
454 }
455
hpre_rsa_cb(struct hpre_ctx * ctx,void * resp)456 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
457 {
458 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
459 struct hpre_asym_request *req;
460 struct akcipher_request *areq;
461 u64 overtime_thrhld;
462 int ret;
463
464 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
465
466 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
467 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
468 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
469
470 areq = req->areq.rsa;
471 areq->dst_len = ctx->key_sz;
472 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
473 akcipher_request_complete(areq, ret);
474 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
475 }
476
hpre_alg_cb(struct hisi_qp * qp,void * resp)477 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
478 {
479 struct hpre_ctx *ctx = qp->qp_ctx;
480 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
481 struct hpre_sqe *sqe = resp;
482 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
483
484 if (unlikely(!req)) {
485 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
486 return;
487 }
488
489 req->cb(ctx, resp);
490 }
491
hpre_stop_qp_and_put(struct hisi_qp * qp)492 static void hpre_stop_qp_and_put(struct hisi_qp *qp)
493 {
494 hisi_qm_stop_qp(qp);
495 hisi_qm_free_qps(&qp, 1);
496 }
497
hpre_ctx_init(struct hpre_ctx * ctx,u8 type)498 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
499 {
500 struct hisi_qp *qp;
501 int ret;
502
503 qp = hpre_get_qp_and_start(type);
504 if (IS_ERR(qp))
505 return PTR_ERR(qp);
506
507 qp->qp_ctx = ctx;
508 qp->req_cb = hpre_alg_cb;
509
510 ret = hpre_ctx_set(ctx, qp, qp->sq_depth);
511 if (ret)
512 hpre_stop_qp_and_put(qp);
513
514 return ret;
515 }
516
hpre_msg_request_set(struct hpre_ctx * ctx,void * req,bool is_rsa)517 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
518 {
519 struct hpre_asym_request *h_req;
520 struct hpre_sqe *msg;
521 int req_id;
522 void *tmp;
523
524 if (is_rsa) {
525 struct akcipher_request *akreq = req;
526
527 if (akreq->dst_len < ctx->key_sz) {
528 akreq->dst_len = ctx->key_sz;
529 return -EOVERFLOW;
530 }
531
532 tmp = akcipher_request_ctx(akreq);
533 h_req = PTR_ALIGN(tmp, hpre_align_sz());
534 h_req->cb = hpre_rsa_cb;
535 h_req->areq.rsa = akreq;
536 msg = &h_req->req;
537 memset(msg, 0, sizeof(*msg));
538 } else {
539 struct kpp_request *kreq = req;
540
541 if (kreq->dst_len < ctx->key_sz) {
542 kreq->dst_len = ctx->key_sz;
543 return -EOVERFLOW;
544 }
545
546 tmp = kpp_request_ctx(kreq);
547 h_req = PTR_ALIGN(tmp, hpre_align_sz());
548 h_req->cb = hpre_dh_cb;
549 h_req->areq.dh = kreq;
550 msg = &h_req->req;
551 memset(msg, 0, sizeof(*msg));
552 msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
553 }
554
555 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
556 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
557 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
558 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
559 h_req->ctx = ctx;
560
561 req_id = hpre_add_req_to_ctx(h_req);
562 if (req_id < 0)
563 return -EBUSY;
564
565 msg->tag = cpu_to_le16((u16)req_id);
566
567 return 0;
568 }
569
hpre_send(struct hpre_ctx * ctx,struct hpre_sqe * msg)570 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
571 {
572 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
573 int ctr = 0;
574 int ret;
575
576 do {
577 atomic64_inc(&dfx[HPRE_SEND_CNT].value);
578 spin_lock_bh(&ctx->req_lock);
579 ret = hisi_qp_send(ctx->qp, msg);
580 spin_unlock_bh(&ctx->req_lock);
581 if (ret != -EBUSY)
582 break;
583 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
584 } while (ctr++ < HPRE_TRY_SEND_TIMES);
585
586 if (likely(!ret))
587 return ret;
588
589 if (ret != -EBUSY)
590 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
591
592 return ret;
593 }
594
hpre_dh_compute_value(struct kpp_request * req)595 static int hpre_dh_compute_value(struct kpp_request *req)
596 {
597 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
598 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
599 void *tmp = kpp_request_ctx(req);
600 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
601 struct hpre_sqe *msg = &hpre_req->req;
602 int ret;
603
604 ret = hpre_msg_request_set(ctx, req, false);
605 if (unlikely(ret))
606 return ret;
607
608 if (req->src) {
609 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
610 if (unlikely(ret))
611 goto clear_all;
612 } else {
613 msg->in = cpu_to_le64(ctx->dh.dma_g);
614 }
615
616 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
617 if (unlikely(ret))
618 goto clear_all;
619
620 if (ctx->crt_g2_mode && !req->src)
621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
622 else
623 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
624
625 /* success */
626 ret = hpre_send(ctx, msg);
627 if (likely(!ret))
628 return -EINPROGRESS;
629
630 clear_all:
631 hpre_rm_req_from_ctx(hpre_req);
632 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
633
634 return ret;
635 }
636
hpre_is_dh_params_length_valid(unsigned int key_sz)637 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
638 {
639 #define _HPRE_DH_GRP1 768
640 #define _HPRE_DH_GRP2 1024
641 #define _HPRE_DH_GRP5 1536
642 #define _HPRE_DH_GRP14 2048
643 #define _HPRE_DH_GRP15 3072
644 #define _HPRE_DH_GRP16 4096
645 switch (key_sz) {
646 case _HPRE_DH_GRP1:
647 case _HPRE_DH_GRP2:
648 case _HPRE_DH_GRP5:
649 case _HPRE_DH_GRP14:
650 case _HPRE_DH_GRP15:
651 case _HPRE_DH_GRP16:
652 return 0;
653 default:
654 return -EINVAL;
655 }
656 }
657
hpre_dh_set_params(struct hpre_ctx * ctx,struct dh * params)658 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
659 {
660 struct device *dev = ctx->dev;
661 unsigned int sz;
662
663 if (params->p_size > HPRE_DH_MAX_P_SZ)
664 return -EINVAL;
665
666 if (hpre_is_dh_params_length_valid(params->p_size <<
667 HPRE_BITS_2_BYTES_SHIFT))
668 return -EINVAL;
669
670 sz = ctx->key_sz = params->p_size;
671 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
672 &ctx->dh.dma_xa_p, GFP_KERNEL);
673 if (!ctx->dh.xa_p)
674 return -ENOMEM;
675
676 memcpy(ctx->dh.xa_p + sz, params->p, sz);
677
678 /* If g equals 2 don't copy it */
679 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
680 ctx->crt_g2_mode = true;
681 return 0;
682 }
683
684 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
685 if (!ctx->dh.g) {
686 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
687 ctx->dh.dma_xa_p);
688 ctx->dh.xa_p = NULL;
689 return -ENOMEM;
690 }
691
692 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
693
694 return 0;
695 }
696
hpre_dh_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)697 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
698 {
699 struct device *dev = ctx->dev;
700 unsigned int sz = ctx->key_sz;
701
702 if (is_clear_all)
703 hisi_qm_stop_qp(ctx->qp);
704
705 if (ctx->dh.g) {
706 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
707 ctx->dh.g = NULL;
708 }
709
710 if (ctx->dh.xa_p) {
711 memzero_explicit(ctx->dh.xa_p, sz);
712 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
713 ctx->dh.dma_xa_p);
714 ctx->dh.xa_p = NULL;
715 }
716
717 hpre_ctx_clear(ctx, is_clear_all);
718 }
719
hpre_dh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)720 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
721 unsigned int len)
722 {
723 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
724 struct dh params;
725 int ret;
726
727 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)
728 return -EINVAL;
729
730 /* Free old secret if any */
731 hpre_dh_clear_ctx(ctx, false);
732
733 ret = hpre_dh_set_params(ctx, ¶ms);
734 if (ret < 0)
735 goto err_clear_ctx;
736
737 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
738 params.key_size);
739
740 return 0;
741
742 err_clear_ctx:
743 hpre_dh_clear_ctx(ctx, false);
744 return ret;
745 }
746
hpre_dh_max_size(struct crypto_kpp * tfm)747 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
748 {
749 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
750
751 return ctx->key_sz;
752 }
753
hpre_dh_init_tfm(struct crypto_kpp * tfm)754 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
755 {
756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
757
758 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
759
760 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
761 }
762
hpre_dh_exit_tfm(struct crypto_kpp * tfm)763 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
764 {
765 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
766
767 hpre_dh_clear_ctx(ctx, true);
768 }
769
hpre_rsa_drop_leading_zeros(const char ** ptr,size_t * len)770 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
771 {
772 while (!**ptr && *len) {
773 (*ptr)++;
774 (*len)--;
775 }
776 }
777
hpre_rsa_key_size_is_support(unsigned int len)778 static bool hpre_rsa_key_size_is_support(unsigned int len)
779 {
780 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
781
782 #define _RSA_1024BITS_KEY_WDTH 1024
783 #define _RSA_2048BITS_KEY_WDTH 2048
784 #define _RSA_3072BITS_KEY_WDTH 3072
785 #define _RSA_4096BITS_KEY_WDTH 4096
786
787 switch (bits) {
788 case _RSA_1024BITS_KEY_WDTH:
789 case _RSA_2048BITS_KEY_WDTH:
790 case _RSA_3072BITS_KEY_WDTH:
791 case _RSA_4096BITS_KEY_WDTH:
792 return true;
793 default:
794 return false;
795 }
796 }
797
hpre_rsa_enc(struct akcipher_request * req)798 static int hpre_rsa_enc(struct akcipher_request *req)
799 {
800 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
801 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
802 void *tmp = akcipher_request_ctx(req);
803 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
804 struct hpre_sqe *msg = &hpre_req->req;
805 int ret;
806
807 /* For 512 and 1536 bits key size, use soft tfm instead */
808 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
809 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
810 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
811 ret = crypto_akcipher_encrypt(req);
812 akcipher_request_set_tfm(req, tfm);
813 return ret;
814 }
815
816 if (unlikely(!ctx->rsa.pubkey))
817 return -EINVAL;
818
819 ret = hpre_msg_request_set(ctx, req, true);
820 if (unlikely(ret))
821 return ret;
822
823 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
824 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
825
826 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
827 if (unlikely(ret))
828 goto clear_all;
829
830 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
831 if (unlikely(ret))
832 goto clear_all;
833
834 /* success */
835 ret = hpre_send(ctx, msg);
836 if (likely(!ret))
837 return -EINPROGRESS;
838
839 clear_all:
840 hpre_rm_req_from_ctx(hpre_req);
841 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
842
843 return ret;
844 }
845
hpre_rsa_dec(struct akcipher_request * req)846 static int hpre_rsa_dec(struct akcipher_request *req)
847 {
848 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
849 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
850 void *tmp = akcipher_request_ctx(req);
851 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
852 struct hpre_sqe *msg = &hpre_req->req;
853 int ret;
854
855 /* For 512 and 1536 bits key size, use soft tfm instead */
856 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
857 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
858 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
859 ret = crypto_akcipher_decrypt(req);
860 akcipher_request_set_tfm(req, tfm);
861 return ret;
862 }
863
864 if (unlikely(!ctx->rsa.prikey))
865 return -EINVAL;
866
867 ret = hpre_msg_request_set(ctx, req, true);
868 if (unlikely(ret))
869 return ret;
870
871 if (ctx->crt_g2_mode) {
872 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
873 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
874 HPRE_ALG_NC_CRT);
875 } else {
876 msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
877 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
878 HPRE_ALG_NC_NCRT);
879 }
880
881 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
882 if (unlikely(ret))
883 goto clear_all;
884
885 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
886 if (unlikely(ret))
887 goto clear_all;
888
889 /* success */
890 ret = hpre_send(ctx, msg);
891 if (likely(!ret))
892 return -EINPROGRESS;
893
894 clear_all:
895 hpre_rm_req_from_ctx(hpre_req);
896 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
897
898 return ret;
899 }
900
hpre_rsa_set_n(struct hpre_ctx * ctx,const char * value,size_t vlen,bool private)901 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
902 size_t vlen, bool private)
903 {
904 const char *ptr = value;
905
906 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
907
908 ctx->key_sz = vlen;
909
910 /* if invalid key size provided, we use software tfm */
911 if (!hpre_rsa_key_size_is_support(ctx->key_sz))
912 return 0;
913
914 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
915 &ctx->rsa.dma_pubkey,
916 GFP_KERNEL);
917 if (!ctx->rsa.pubkey)
918 return -ENOMEM;
919
920 if (private) {
921 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
922 &ctx->rsa.dma_prikey,
923 GFP_KERNEL);
924 if (!ctx->rsa.prikey) {
925 dma_free_coherent(ctx->dev, vlen << 1,
926 ctx->rsa.pubkey,
927 ctx->rsa.dma_pubkey);
928 ctx->rsa.pubkey = NULL;
929 return -ENOMEM;
930 }
931 memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
932 }
933 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
934
935 /* Using hardware HPRE to do RSA */
936 return 1;
937 }
938
hpre_rsa_set_e(struct hpre_ctx * ctx,const char * value,size_t vlen)939 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
940 size_t vlen)
941 {
942 const char *ptr = value;
943
944 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
945
946 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
947 return -EINVAL;
948
949 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
950
951 return 0;
952 }
953
hpre_rsa_set_d(struct hpre_ctx * ctx,const char * value,size_t vlen)954 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
955 size_t vlen)
956 {
957 const char *ptr = value;
958
959 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
960
961 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
962 return -EINVAL;
963
964 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
965
966 return 0;
967 }
968
hpre_crt_para_get(char * para,size_t para_sz,const char * raw,size_t raw_sz)969 static int hpre_crt_para_get(char *para, size_t para_sz,
970 const char *raw, size_t raw_sz)
971 {
972 const char *ptr = raw;
973 size_t len = raw_sz;
974
975 hpre_rsa_drop_leading_zeros(&ptr, &len);
976 if (!len || len > para_sz)
977 return -EINVAL;
978
979 memcpy(para + para_sz - len, ptr, len);
980
981 return 0;
982 }
983
hpre_rsa_setkey_crt(struct hpre_ctx * ctx,struct rsa_key * rsa_key)984 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
985 {
986 unsigned int hlf_ksz = ctx->key_sz >> 1;
987 struct device *dev = ctx->dev;
988 u64 offset;
989 int ret;
990
991 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
992 &ctx->rsa.dma_crt_prikey,
993 GFP_KERNEL);
994 if (!ctx->rsa.crt_prikey)
995 return -ENOMEM;
996
997 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
998 rsa_key->dq, rsa_key->dq_sz);
999 if (ret)
1000 goto free_key;
1001
1002 offset = hlf_ksz;
1003 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1004 rsa_key->dp, rsa_key->dp_sz);
1005 if (ret)
1006 goto free_key;
1007
1008 offset = hlf_ksz * HPRE_CRT_Q;
1009 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1010 rsa_key->q, rsa_key->q_sz);
1011 if (ret)
1012 goto free_key;
1013
1014 offset = hlf_ksz * HPRE_CRT_P;
1015 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1016 rsa_key->p, rsa_key->p_sz);
1017 if (ret)
1018 goto free_key;
1019
1020 offset = hlf_ksz * HPRE_CRT_INV;
1021 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1022 rsa_key->qinv, rsa_key->qinv_sz);
1023 if (ret)
1024 goto free_key;
1025
1026 ctx->crt_g2_mode = true;
1027
1028 return 0;
1029
1030 free_key:
1031 offset = hlf_ksz * HPRE_CRT_PRMS;
1032 memzero_explicit(ctx->rsa.crt_prikey, offset);
1033 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1034 ctx->rsa.dma_crt_prikey);
1035 ctx->rsa.crt_prikey = NULL;
1036 ctx->crt_g2_mode = false;
1037
1038 return ret;
1039 }
1040
1041 /* If it is clear all, all the resources of the QP will be cleaned. */
hpre_rsa_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)1042 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1043 {
1044 unsigned int half_key_sz = ctx->key_sz >> 1;
1045 struct device *dev = ctx->dev;
1046
1047 if (is_clear_all)
1048 hisi_qm_stop_qp(ctx->qp);
1049
1050 if (ctx->rsa.pubkey) {
1051 dma_free_coherent(dev, ctx->key_sz << 1,
1052 ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1053 ctx->rsa.pubkey = NULL;
1054 }
1055
1056 if (ctx->rsa.crt_prikey) {
1057 memzero_explicit(ctx->rsa.crt_prikey,
1058 half_key_sz * HPRE_CRT_PRMS);
1059 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1060 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1061 ctx->rsa.crt_prikey = NULL;
1062 }
1063
1064 if (ctx->rsa.prikey) {
1065 memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1066 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1067 ctx->rsa.dma_prikey);
1068 ctx->rsa.prikey = NULL;
1069 }
1070
1071 hpre_ctx_clear(ctx, is_clear_all);
1072 }
1073
1074 /*
1075 * we should judge if it is CRT or not,
1076 * CRT: return true, N-CRT: return false .
1077 */
hpre_is_crt_key(struct rsa_key * key)1078 static bool hpre_is_crt_key(struct rsa_key *key)
1079 {
1080 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1081 key->qinv_sz;
1082
1083 #define LEN_OF_NCRT_PARA 5
1084
1085 /* N-CRT less than 5 parameters */
1086 return len > LEN_OF_NCRT_PARA;
1087 }
1088
hpre_rsa_setkey(struct hpre_ctx * ctx,const void * key,unsigned int keylen,bool private)1089 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1090 unsigned int keylen, bool private)
1091 {
1092 struct rsa_key rsa_key;
1093 int ret;
1094
1095 hpre_rsa_clear_ctx(ctx, false);
1096
1097 if (private)
1098 ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1099 else
1100 ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1101 if (ret < 0)
1102 return ret;
1103
1104 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1105 if (ret <= 0)
1106 return ret;
1107
1108 if (private) {
1109 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1110 if (ret < 0)
1111 goto free;
1112
1113 if (hpre_is_crt_key(&rsa_key)) {
1114 ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1115 if (ret < 0)
1116 goto free;
1117 }
1118 }
1119
1120 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1121 if (ret < 0)
1122 goto free;
1123
1124 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1125 ret = -EINVAL;
1126 goto free;
1127 }
1128
1129 return 0;
1130
1131 free:
1132 hpre_rsa_clear_ctx(ctx, false);
1133 return ret;
1134 }
1135
hpre_rsa_setpubkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1136 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1137 unsigned int keylen)
1138 {
1139 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1140 int ret;
1141
1142 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1143 if (ret)
1144 return ret;
1145
1146 return hpre_rsa_setkey(ctx, key, keylen, false);
1147 }
1148
hpre_rsa_setprivkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1149 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1150 unsigned int keylen)
1151 {
1152 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1153 int ret;
1154
1155 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1156 if (ret)
1157 return ret;
1158
1159 return hpre_rsa_setkey(ctx, key, keylen, true);
1160 }
1161
hpre_rsa_max_size(struct crypto_akcipher * tfm)1162 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1163 {
1164 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1165
1166 /* For 512 and 1536 bits key size, use soft tfm instead */
1167 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1168 ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1169 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1170
1171 return ctx->key_sz;
1172 }
1173
hpre_rsa_init_tfm(struct crypto_akcipher * tfm)1174 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1175 {
1176 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1177 int ret;
1178
1179 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1180 if (IS_ERR(ctx->rsa.soft_tfm)) {
1181 pr_err("Can not alloc_akcipher!\n");
1182 return PTR_ERR(ctx->rsa.soft_tfm);
1183 }
1184
1185 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +
1186 hpre_align_pd());
1187
1188 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1189 if (ret)
1190 crypto_free_akcipher(ctx->rsa.soft_tfm);
1191
1192 return ret;
1193 }
1194
hpre_rsa_exit_tfm(struct crypto_akcipher * tfm)1195 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1196 {
1197 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1198
1199 hpre_rsa_clear_ctx(ctx, true);
1200 crypto_free_akcipher(ctx->rsa.soft_tfm);
1201 }
1202
hpre_key_to_big_end(u8 * data,int len)1203 static void hpre_key_to_big_end(u8 *data, int len)
1204 {
1205 int i, j;
1206
1207 for (i = 0; i < len / 2; i++) {
1208 j = len - i - 1;
1209 swap(data[j], data[i]);
1210 }
1211 }
1212
hpre_ecc_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all,bool is_ecdh)1213 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
1214 bool is_ecdh)
1215 {
1216 struct device *dev = ctx->dev;
1217 unsigned int sz = ctx->key_sz;
1218 unsigned int shift = sz << 1;
1219
1220 if (is_clear_all)
1221 hisi_qm_stop_qp(ctx->qp);
1222
1223 if (is_ecdh && ctx->ecdh.p) {
1224 /* ecdh: p->a->k->b */
1225 memzero_explicit(ctx->ecdh.p + shift, sz);
1226 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1227 ctx->ecdh.p = NULL;
1228 } else if (!is_ecdh && ctx->curve25519.p) {
1229 /* curve25519: p->a->k */
1230 memzero_explicit(ctx->curve25519.p + shift, sz);
1231 dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
1232 ctx->curve25519.dma_p);
1233 ctx->curve25519.p = NULL;
1234 }
1235
1236 hpre_ctx_clear(ctx, is_clear_all);
1237 }
1238
1239 /*
1240 * The bits of 192/224/256/384/521 are supported by HPRE,
1241 * and convert the bits like:
1242 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
1243 * If the parameter bit width is insufficient, then we fill in the
1244 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
1245 */
hpre_ecdh_supported_curve(unsigned short id)1246 static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1247 {
1248 switch (id) {
1249 case ECC_CURVE_NIST_P192:
1250 case ECC_CURVE_NIST_P256:
1251 return HPRE_ECC_HW256_KSZ_B;
1252 case ECC_CURVE_NIST_P384:
1253 return HPRE_ECC_HW384_KSZ_B;
1254 default:
1255 break;
1256 }
1257
1258 return 0;
1259 }
1260
fill_curve_param(void * addr,u64 * param,unsigned int cur_sz,u8 ndigits)1261 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1262 {
1263 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1264 u8 i = 0;
1265
1266 while (i < ndigits - 1) {
1267 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64));
1268 i++;
1269 }
1270
1271 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz);
1272 hpre_key_to_big_end((u8 *)addr, cur_sz);
1273 }
1274
hpre_ecdh_fill_curve(struct hpre_ctx * ctx,struct ecdh * params,unsigned int cur_sz)1275 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1276 unsigned int cur_sz)
1277 {
1278 unsigned int shifta = ctx->key_sz << 1;
1279 unsigned int shiftb = ctx->key_sz << 2;
1280 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1281 void *a = ctx->ecdh.p + shifta - cur_sz;
1282 void *b = ctx->ecdh.p + shiftb - cur_sz;
1283 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1284 void *y = ctx->ecdh.g + shifta - cur_sz;
1285 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1286 char *n;
1287
1288 if (unlikely(!curve))
1289 return -EINVAL;
1290
1291 n = kzalloc(ctx->key_sz, GFP_KERNEL);
1292 if (!n)
1293 return -ENOMEM;
1294
1295 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1296 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1297 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1298 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1299 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1300 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1301
1302 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1303 kfree(n);
1304 return -EINVAL;
1305 }
1306
1307 kfree(n);
1308 return 0;
1309 }
1310
hpre_ecdh_get_curvesz(unsigned short id)1311 static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1312 {
1313 switch (id) {
1314 case ECC_CURVE_NIST_P192:
1315 return HPRE_ECC_NIST_P192_N_SIZE;
1316 case ECC_CURVE_NIST_P256:
1317 return HPRE_ECC_NIST_P256_N_SIZE;
1318 case ECC_CURVE_NIST_P384:
1319 return HPRE_ECC_NIST_P384_N_SIZE;
1320 default:
1321 break;
1322 }
1323
1324 return 0;
1325 }
1326
hpre_ecdh_set_param(struct hpre_ctx * ctx,struct ecdh * params)1327 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1328 {
1329 struct device *dev = ctx->dev;
1330 unsigned int sz, shift, curve_sz;
1331 int ret;
1332
1333 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1334 if (!ctx->key_sz)
1335 return -EINVAL;
1336
1337 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1338 if (!curve_sz || params->key_size > curve_sz)
1339 return -EINVAL;
1340
1341 sz = ctx->key_sz;
1342
1343 if (!ctx->ecdh.p) {
1344 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1345 GFP_KERNEL);
1346 if (!ctx->ecdh.p)
1347 return -ENOMEM;
1348 }
1349
1350 shift = sz << 2;
1351 ctx->ecdh.g = ctx->ecdh.p + shift;
1352 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1353
1354 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1355 if (ret) {
1356 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1357 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1358 ctx->ecdh.p = NULL;
1359 return ret;
1360 }
1361
1362 return 0;
1363 }
1364
hpre_key_is_zero(char * key,unsigned short key_sz)1365 static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1366 {
1367 int i;
1368
1369 for (i = 0; i < key_sz; i++)
1370 if (key[i])
1371 return false;
1372
1373 return true;
1374 }
1375
ecdh_gen_privkey(struct hpre_ctx * ctx,struct ecdh * params)1376 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
1377 {
1378 struct device *dev = ctx->dev;
1379 int ret;
1380
1381 ret = crypto_get_default_rng();
1382 if (ret) {
1383 dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
1384 return ret;
1385 }
1386
1387 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
1388 params->key_size);
1389 crypto_put_default_rng();
1390 if (ret)
1391 dev_err(dev, "failed to get rng, ret = %d!\n", ret);
1392
1393 return ret;
1394 }
1395
hpre_ecdh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1396 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1397 unsigned int len)
1398 {
1399 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1400 unsigned int sz, sz_shift, curve_sz;
1401 struct device *dev = ctx->dev;
1402 char key[HPRE_ECC_MAX_KSZ];
1403 struct ecdh params;
1404 int ret;
1405
1406 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) {
1407 dev_err(dev, "failed to decode ecdh key!\n");
1408 return -EINVAL;
1409 }
1410
1411 /* Use stdrng to generate private key */
1412 if (!params.key || !params.key_size) {
1413 params.key = key;
1414 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1415 if (!curve_sz) {
1416 dev_err(dev, "Invalid curve size!\n");
1417 return -EINVAL;
1418 }
1419
1420 params.key_size = curve_sz - 1;
1421 ret = ecdh_gen_privkey(ctx, ¶ms);
1422 if (ret)
1423 return ret;
1424 }
1425
1426 if (hpre_key_is_zero(params.key, params.key_size)) {
1427 dev_err(dev, "Invalid hpre key!\n");
1428 return -EINVAL;
1429 }
1430
1431 hpre_ecc_clear_ctx(ctx, false, true);
1432
1433 ret = hpre_ecdh_set_param(ctx, ¶ms);
1434 if (ret < 0) {
1435 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1436 return ret;
1437 }
1438
1439 sz = ctx->key_sz;
1440 sz_shift = (sz << 1) + sz - params.key_size;
1441 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1442
1443 return 0;
1444 }
1445
hpre_ecdh_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1446 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1447 struct hpre_asym_request *req,
1448 struct scatterlist *dst,
1449 struct scatterlist *src)
1450 {
1451 struct device *dev = ctx->dev;
1452 struct hpre_sqe *sqe = &req->req;
1453 dma_addr_t dma;
1454
1455 dma = le64_to_cpu(sqe->in);
1456 if (unlikely(dma_mapping_error(dev, dma)))
1457 return;
1458
1459 if (src && req->src)
1460 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1461
1462 dma = le64_to_cpu(sqe->out);
1463 if (unlikely(dma_mapping_error(dev, dma)))
1464 return;
1465
1466 if (req->dst)
1467 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1468 if (dst)
1469 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1470 }
1471
hpre_ecdh_cb(struct hpre_ctx * ctx,void * resp)1472 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1473 {
1474 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1475 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1476 struct hpre_asym_request *req = NULL;
1477 struct kpp_request *areq;
1478 u64 overtime_thrhld;
1479 char *p;
1480 int ret;
1481
1482 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1483 areq = req->areq.ecdh;
1484 areq->dst_len = ctx->key_sz << 1;
1485
1486 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1487 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1488 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1489
1490 p = sg_virt(areq->dst);
1491 memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1492 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1493
1494 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1495 kpp_request_complete(areq, ret);
1496
1497 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1498 }
1499
hpre_ecdh_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1500 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1501 struct kpp_request *req)
1502 {
1503 struct hpre_asym_request *h_req;
1504 struct hpre_sqe *msg;
1505 int req_id;
1506 void *tmp;
1507
1508 if (req->dst_len < ctx->key_sz << 1) {
1509 req->dst_len = ctx->key_sz << 1;
1510 return -EINVAL;
1511 }
1512
1513 tmp = kpp_request_ctx(req);
1514 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1515 h_req->cb = hpre_ecdh_cb;
1516 h_req->areq.ecdh = req;
1517 msg = &h_req->req;
1518 memset(msg, 0, sizeof(*msg));
1519 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1520 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1521 msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1522
1523 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1524 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1525 h_req->ctx = ctx;
1526
1527 req_id = hpre_add_req_to_ctx(h_req);
1528 if (req_id < 0)
1529 return -EBUSY;
1530
1531 msg->tag = cpu_to_le16((u16)req_id);
1532 return 0;
1533 }
1534
hpre_ecdh_src_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1535 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1536 struct scatterlist *data, unsigned int len)
1537 {
1538 struct hpre_sqe *msg = &hpre_req->req;
1539 struct hpre_ctx *ctx = hpre_req->ctx;
1540 struct device *dev = ctx->dev;
1541 unsigned int tmpshift;
1542 dma_addr_t dma = 0;
1543 void *ptr;
1544 int shift;
1545
1546 /* Src_data include gx and gy. */
1547 shift = ctx->key_sz - (len >> 1);
1548 if (unlikely(shift < 0))
1549 return -EINVAL;
1550
1551 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1552 if (unlikely(!ptr))
1553 return -ENOMEM;
1554
1555 tmpshift = ctx->key_sz << 1;
1556 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1557 memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1558 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1559
1560 hpre_req->src = ptr;
1561 msg->in = cpu_to_le64(dma);
1562 return 0;
1563 }
1564
hpre_ecdh_dst_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1565 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1566 struct scatterlist *data, unsigned int len)
1567 {
1568 struct hpre_sqe *msg = &hpre_req->req;
1569 struct hpre_ctx *ctx = hpre_req->ctx;
1570 struct device *dev = ctx->dev;
1571 dma_addr_t dma;
1572
1573 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1574 dev_err(dev, "data or data length is illegal!\n");
1575 return -EINVAL;
1576 }
1577
1578 hpre_req->dst = NULL;
1579 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1580 if (unlikely(dma_mapping_error(dev, dma))) {
1581 dev_err(dev, "dma map data err!\n");
1582 return -ENOMEM;
1583 }
1584
1585 msg->out = cpu_to_le64(dma);
1586 return 0;
1587 }
1588
hpre_ecdh_compute_value(struct kpp_request * req)1589 static int hpre_ecdh_compute_value(struct kpp_request *req)
1590 {
1591 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1592 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1593 struct device *dev = ctx->dev;
1594 void *tmp = kpp_request_ctx(req);
1595 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1596 struct hpre_sqe *msg = &hpre_req->req;
1597 int ret;
1598
1599 ret = hpre_ecdh_msg_request_set(ctx, req);
1600 if (unlikely(ret)) {
1601 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1602 return ret;
1603 }
1604
1605 if (req->src) {
1606 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1607 if (unlikely(ret)) {
1608 dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1609 goto clear_all;
1610 }
1611 } else {
1612 msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1613 }
1614
1615 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1616 if (unlikely(ret)) {
1617 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1618 goto clear_all;
1619 }
1620
1621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1622 ret = hpre_send(ctx, msg);
1623 if (likely(!ret))
1624 return -EINPROGRESS;
1625
1626 clear_all:
1627 hpre_rm_req_from_ctx(hpre_req);
1628 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1629 return ret;
1630 }
1631
hpre_ecdh_max_size(struct crypto_kpp * tfm)1632 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1633 {
1634 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1635
1636 /* max size is the pub_key_size, include x and y */
1637 return ctx->key_sz << 1;
1638 }
1639
hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp * tfm)1640 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1641 {
1642 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1643
1644 ctx->curve_id = ECC_CURVE_NIST_P192;
1645
1646 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1647
1648 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1649 }
1650
hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp * tfm)1651 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1652 {
1653 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1654
1655 ctx->curve_id = ECC_CURVE_NIST_P256;
1656
1657 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1658
1659 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1660 }
1661
hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp * tfm)1662 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
1663 {
1664 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1665
1666 ctx->curve_id = ECC_CURVE_NIST_P384;
1667
1668 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1669
1670 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1671 }
1672
hpre_ecdh_exit_tfm(struct crypto_kpp * tfm)1673 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1674 {
1675 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1676
1677 hpre_ecc_clear_ctx(ctx, true, true);
1678 }
1679
hpre_curve25519_fill_curve(struct hpre_ctx * ctx,const void * buf,unsigned int len)1680 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
1681 unsigned int len)
1682 {
1683 u8 secret[CURVE25519_KEY_SIZE] = { 0 };
1684 unsigned int sz = ctx->key_sz;
1685 const struct ecc_curve *curve;
1686 unsigned int shift = sz << 1;
1687 void *p;
1688
1689 /*
1690 * The key from 'buf' is in little-endian, we should preprocess it as
1691 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
1692 * then convert it to big endian. Only in this way, the result can be
1693 * the same as the software curve-25519 that exists in crypto.
1694 */
1695 memcpy(secret, buf, len);
1696 curve25519_clamp_secret(secret);
1697 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
1698
1699 p = ctx->curve25519.p + sz - len;
1700
1701 curve = ecc_get_curve25519();
1702
1703 /* fill curve parameters */
1704 fill_curve_param(p, curve->p, len, curve->g.ndigits);
1705 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
1706 memcpy(p + shift, secret, len);
1707 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
1708 memzero_explicit(secret, CURVE25519_KEY_SIZE);
1709 }
1710
hpre_curve25519_set_param(struct hpre_ctx * ctx,const void * buf,unsigned int len)1711 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
1712 unsigned int len)
1713 {
1714 struct device *dev = ctx->dev;
1715 unsigned int sz = ctx->key_sz;
1716 unsigned int shift = sz << 1;
1717
1718 /* p->a->k->gx */
1719 if (!ctx->curve25519.p) {
1720 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
1721 &ctx->curve25519.dma_p,
1722 GFP_KERNEL);
1723 if (!ctx->curve25519.p)
1724 return -ENOMEM;
1725 }
1726
1727 ctx->curve25519.g = ctx->curve25519.p + shift + sz;
1728 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
1729
1730 hpre_curve25519_fill_curve(ctx, buf, len);
1731
1732 return 0;
1733 }
1734
hpre_curve25519_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1735 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1736 unsigned int len)
1737 {
1738 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1739 struct device *dev = ctx->dev;
1740 int ret = -EINVAL;
1741
1742 if (len != CURVE25519_KEY_SIZE ||
1743 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1744 dev_err(dev, "key is null or key len is not 32bytes!\n");
1745 return ret;
1746 }
1747
1748 /* Free old secret if any */
1749 hpre_ecc_clear_ctx(ctx, false, false);
1750
1751 ctx->key_sz = CURVE25519_KEY_SIZE;
1752 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
1753 if (ret) {
1754 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
1755 hpre_ecc_clear_ctx(ctx, false, false);
1756 return ret;
1757 }
1758
1759 return 0;
1760 }
1761
hpre_curve25519_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1762 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
1763 struct hpre_asym_request *req,
1764 struct scatterlist *dst,
1765 struct scatterlist *src)
1766 {
1767 struct device *dev = ctx->dev;
1768 struct hpre_sqe *sqe = &req->req;
1769 dma_addr_t dma;
1770
1771 dma = le64_to_cpu(sqe->in);
1772 if (unlikely(dma_mapping_error(dev, dma)))
1773 return;
1774
1775 if (src && req->src)
1776 dma_free_coherent(dev, ctx->key_sz, req->src, dma);
1777
1778 dma = le64_to_cpu(sqe->out);
1779 if (unlikely(dma_mapping_error(dev, dma)))
1780 return;
1781
1782 if (req->dst)
1783 dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
1784 if (dst)
1785 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
1786 }
1787
hpre_curve25519_cb(struct hpre_ctx * ctx,void * resp)1788 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
1789 {
1790 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1791 struct hpre_asym_request *req = NULL;
1792 struct kpp_request *areq;
1793 u64 overtime_thrhld;
1794 int ret;
1795
1796 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1797 areq = req->areq.curve25519;
1798 areq->dst_len = ctx->key_sz;
1799
1800 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1801 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1802 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1803
1804 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1805
1806 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1807 kpp_request_complete(areq, ret);
1808
1809 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1810 }
1811
hpre_curve25519_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1812 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
1813 struct kpp_request *req)
1814 {
1815 struct hpre_asym_request *h_req;
1816 struct hpre_sqe *msg;
1817 int req_id;
1818 void *tmp;
1819
1820 if (unlikely(req->dst_len < ctx->key_sz)) {
1821 req->dst_len = ctx->key_sz;
1822 return -EINVAL;
1823 }
1824
1825 tmp = kpp_request_ctx(req);
1826 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1827 h_req->cb = hpre_curve25519_cb;
1828 h_req->areq.curve25519 = req;
1829 msg = &h_req->req;
1830 memset(msg, 0, sizeof(*msg));
1831 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1832 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1833 msg->key = cpu_to_le64(ctx->curve25519.dma_p);
1834
1835 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1836 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1837 h_req->ctx = ctx;
1838
1839 req_id = hpre_add_req_to_ctx(h_req);
1840 if (req_id < 0)
1841 return -EBUSY;
1842
1843 msg->tag = cpu_to_le16((u16)req_id);
1844 return 0;
1845 }
1846
hpre_curve25519_src_modulo_p(u8 * ptr)1847 static void hpre_curve25519_src_modulo_p(u8 *ptr)
1848 {
1849 int i;
1850
1851 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
1852 ptr[i] = 0;
1853
1854 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */
1855 ptr[i] -= 0xed;
1856 }
1857
hpre_curve25519_src_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1858 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
1859 struct scatterlist *data, unsigned int len)
1860 {
1861 struct hpre_sqe *msg = &hpre_req->req;
1862 struct hpre_ctx *ctx = hpre_req->ctx;
1863 struct device *dev = ctx->dev;
1864 u8 p[CURVE25519_KEY_SIZE] = { 0 };
1865 const struct ecc_curve *curve;
1866 dma_addr_t dma = 0;
1867 u8 *ptr;
1868
1869 if (len != CURVE25519_KEY_SIZE) {
1870 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
1871 return -EINVAL;
1872 }
1873
1874 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
1875 if (unlikely(!ptr))
1876 return -ENOMEM;
1877
1878 scatterwalk_map_and_copy(ptr, data, 0, len, 0);
1879
1880 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1881 dev_err(dev, "gx is null!\n");
1882 goto err;
1883 }
1884
1885 /*
1886 * Src_data(gx) is in little-endian order, MSB in the final byte should
1887 * be masked as described in RFC7748, then transform it to big-endian
1888 * form, then hisi_hpre can use the data.
1889 */
1890 ptr[31] &= 0x7f;
1891 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
1892
1893 curve = ecc_get_curve25519();
1894
1895 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
1896
1897 /*
1898 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,
1899 * we get its modulus to p, and then use it.
1900 */
1901 if (memcmp(ptr, p, ctx->key_sz) == 0) {
1902 dev_err(dev, "gx is p!\n");
1903 goto err;
1904 } else if (memcmp(ptr, p, ctx->key_sz) > 0) {
1905 hpre_curve25519_src_modulo_p(ptr);
1906 }
1907
1908 hpre_req->src = ptr;
1909 msg->in = cpu_to_le64(dma);
1910 return 0;
1911
1912 err:
1913 dma_free_coherent(dev, ctx->key_sz, ptr, dma);
1914 return -EINVAL;
1915 }
1916
hpre_curve25519_dst_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1917 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
1918 struct scatterlist *data, unsigned int len)
1919 {
1920 struct hpre_sqe *msg = &hpre_req->req;
1921 struct hpre_ctx *ctx = hpre_req->ctx;
1922 struct device *dev = ctx->dev;
1923 dma_addr_t dma;
1924
1925 if (!data || !sg_is_last(data) || len != ctx->key_sz) {
1926 dev_err(dev, "data or data length is illegal!\n");
1927 return -EINVAL;
1928 }
1929
1930 hpre_req->dst = NULL;
1931 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1932 if (unlikely(dma_mapping_error(dev, dma))) {
1933 dev_err(dev, "dma map data err!\n");
1934 return -ENOMEM;
1935 }
1936
1937 msg->out = cpu_to_le64(dma);
1938 return 0;
1939 }
1940
hpre_curve25519_compute_value(struct kpp_request * req)1941 static int hpre_curve25519_compute_value(struct kpp_request *req)
1942 {
1943 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1944 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1945 struct device *dev = ctx->dev;
1946 void *tmp = kpp_request_ctx(req);
1947 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1948 struct hpre_sqe *msg = &hpre_req->req;
1949 int ret;
1950
1951 ret = hpre_curve25519_msg_request_set(ctx, req);
1952 if (unlikely(ret)) {
1953 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
1954 return ret;
1955 }
1956
1957 if (req->src) {
1958 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
1959 if (unlikely(ret)) {
1960 dev_err(dev, "failed to init src data, ret = %d!\n",
1961 ret);
1962 goto clear_all;
1963 }
1964 } else {
1965 msg->in = cpu_to_le64(ctx->curve25519.dma_g);
1966 }
1967
1968 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
1969 if (unlikely(ret)) {
1970 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1971 goto clear_all;
1972 }
1973
1974 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
1975 ret = hpre_send(ctx, msg);
1976 if (likely(!ret))
1977 return -EINPROGRESS;
1978
1979 clear_all:
1980 hpre_rm_req_from_ctx(hpre_req);
1981 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1982 return ret;
1983 }
1984
hpre_curve25519_max_size(struct crypto_kpp * tfm)1985 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
1986 {
1987 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1988
1989 return ctx->key_sz;
1990 }
1991
hpre_curve25519_init_tfm(struct crypto_kpp * tfm)1992 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
1993 {
1994 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1995
1996 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1997
1998 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1999 }
2000
hpre_curve25519_exit_tfm(struct crypto_kpp * tfm)2001 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
2002 {
2003 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
2004
2005 hpre_ecc_clear_ctx(ctx, true, false);
2006 }
2007
2008 static struct akcipher_alg rsa = {
2009 .sign = hpre_rsa_dec,
2010 .verify = hpre_rsa_enc,
2011 .encrypt = hpre_rsa_enc,
2012 .decrypt = hpre_rsa_dec,
2013 .set_pub_key = hpre_rsa_setpubkey,
2014 .set_priv_key = hpre_rsa_setprivkey,
2015 .max_size = hpre_rsa_max_size,
2016 .init = hpre_rsa_init_tfm,
2017 .exit = hpre_rsa_exit_tfm,
2018 .base = {
2019 .cra_ctxsize = sizeof(struct hpre_ctx),
2020 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2021 .cra_name = "rsa",
2022 .cra_driver_name = "hpre-rsa",
2023 .cra_module = THIS_MODULE,
2024 },
2025 };
2026
2027 static struct kpp_alg dh = {
2028 .set_secret = hpre_dh_set_secret,
2029 .generate_public_key = hpre_dh_compute_value,
2030 .compute_shared_secret = hpre_dh_compute_value,
2031 .max_size = hpre_dh_max_size,
2032 .init = hpre_dh_init_tfm,
2033 .exit = hpre_dh_exit_tfm,
2034 .base = {
2035 .cra_ctxsize = sizeof(struct hpre_ctx),
2036 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2037 .cra_name = "dh",
2038 .cra_driver_name = "hpre-dh",
2039 .cra_module = THIS_MODULE,
2040 },
2041 };
2042
2043 static struct kpp_alg ecdh_curves[] = {
2044 {
2045 .set_secret = hpre_ecdh_set_secret,
2046 .generate_public_key = hpre_ecdh_compute_value,
2047 .compute_shared_secret = hpre_ecdh_compute_value,
2048 .max_size = hpre_ecdh_max_size,
2049 .init = hpre_ecdh_nist_p192_init_tfm,
2050 .exit = hpre_ecdh_exit_tfm,
2051 .base = {
2052 .cra_ctxsize = sizeof(struct hpre_ctx),
2053 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2054 .cra_name = "ecdh-nist-p192",
2055 .cra_driver_name = "hpre-ecdh-nist-p192",
2056 .cra_module = THIS_MODULE,
2057 },
2058 }, {
2059 .set_secret = hpre_ecdh_set_secret,
2060 .generate_public_key = hpre_ecdh_compute_value,
2061 .compute_shared_secret = hpre_ecdh_compute_value,
2062 .max_size = hpre_ecdh_max_size,
2063 .init = hpre_ecdh_nist_p256_init_tfm,
2064 .exit = hpre_ecdh_exit_tfm,
2065 .base = {
2066 .cra_ctxsize = sizeof(struct hpre_ctx),
2067 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2068 .cra_name = "ecdh-nist-p256",
2069 .cra_driver_name = "hpre-ecdh-nist-p256",
2070 .cra_module = THIS_MODULE,
2071 },
2072 }, {
2073 .set_secret = hpre_ecdh_set_secret,
2074 .generate_public_key = hpre_ecdh_compute_value,
2075 .compute_shared_secret = hpre_ecdh_compute_value,
2076 .max_size = hpre_ecdh_max_size,
2077 .init = hpre_ecdh_nist_p384_init_tfm,
2078 .exit = hpre_ecdh_exit_tfm,
2079 .base = {
2080 .cra_ctxsize = sizeof(struct hpre_ctx),
2081 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2082 .cra_name = "ecdh-nist-p384",
2083 .cra_driver_name = "hpre-ecdh-nist-p384",
2084 .cra_module = THIS_MODULE,
2085 },
2086 }
2087 };
2088
2089 static struct kpp_alg curve25519_alg = {
2090 .set_secret = hpre_curve25519_set_secret,
2091 .generate_public_key = hpre_curve25519_compute_value,
2092 .compute_shared_secret = hpre_curve25519_compute_value,
2093 .max_size = hpre_curve25519_max_size,
2094 .init = hpre_curve25519_init_tfm,
2095 .exit = hpre_curve25519_exit_tfm,
2096 .base = {
2097 .cra_ctxsize = sizeof(struct hpre_ctx),
2098 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2099 .cra_name = "curve25519",
2100 .cra_driver_name = "hpre-curve25519",
2101 .cra_module = THIS_MODULE,
2102 },
2103 };
2104
hpre_register_rsa(struct hisi_qm * qm)2105 static int hpre_register_rsa(struct hisi_qm *qm)
2106 {
2107 int ret;
2108
2109 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2110 return 0;
2111
2112 rsa.base.cra_flags = 0;
2113 ret = crypto_register_akcipher(&rsa);
2114 if (ret)
2115 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);
2116
2117 return ret;
2118 }
2119
hpre_unregister_rsa(struct hisi_qm * qm)2120 static void hpre_unregister_rsa(struct hisi_qm *qm)
2121 {
2122 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2123 return;
2124
2125 crypto_unregister_akcipher(&rsa);
2126 }
2127
hpre_register_dh(struct hisi_qm * qm)2128 static int hpre_register_dh(struct hisi_qm *qm)
2129 {
2130 int ret;
2131
2132 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2133 return 0;
2134
2135 ret = crypto_register_kpp(&dh);
2136 if (ret)
2137 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);
2138
2139 return ret;
2140 }
2141
hpre_unregister_dh(struct hisi_qm * qm)2142 static void hpre_unregister_dh(struct hisi_qm *qm)
2143 {
2144 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2145 return;
2146
2147 crypto_unregister_kpp(&dh);
2148 }
2149
hpre_register_ecdh(struct hisi_qm * qm)2150 static int hpre_register_ecdh(struct hisi_qm *qm)
2151 {
2152 int ret, i;
2153
2154 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2155 return 0;
2156
2157 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {
2158 ret = crypto_register_kpp(&ecdh_curves[i]);
2159 if (ret) {
2160 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",
2161 ecdh_curves[i].base.cra_name, ret);
2162 goto unreg_kpp;
2163 }
2164 }
2165
2166 return 0;
2167
2168 unreg_kpp:
2169 for (--i; i >= 0; --i)
2170 crypto_unregister_kpp(&ecdh_curves[i]);
2171
2172 return ret;
2173 }
2174
hpre_unregister_ecdh(struct hisi_qm * qm)2175 static void hpre_unregister_ecdh(struct hisi_qm *qm)
2176 {
2177 int i;
2178
2179 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2180 return;
2181
2182 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)
2183 crypto_unregister_kpp(&ecdh_curves[i]);
2184 }
2185
hpre_register_x25519(struct hisi_qm * qm)2186 static int hpre_register_x25519(struct hisi_qm *qm)
2187 {
2188 int ret;
2189
2190 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2191 return 0;
2192
2193 ret = crypto_register_kpp(&curve25519_alg);
2194 if (ret)
2195 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret);
2196
2197 return ret;
2198 }
2199
hpre_unregister_x25519(struct hisi_qm * qm)2200 static void hpre_unregister_x25519(struct hisi_qm *qm)
2201 {
2202 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2203 return;
2204
2205 crypto_unregister_kpp(&curve25519_alg);
2206 }
2207
hpre_algs_register(struct hisi_qm * qm)2208 int hpre_algs_register(struct hisi_qm *qm)
2209 {
2210 int ret = 0;
2211
2212 mutex_lock(&hpre_algs_lock);
2213 if (hpre_available_devs) {
2214 hpre_available_devs++;
2215 goto unlock;
2216 }
2217
2218 ret = hpre_register_rsa(qm);
2219 if (ret)
2220 goto unlock;
2221
2222 ret = hpre_register_dh(qm);
2223 if (ret)
2224 goto unreg_rsa;
2225
2226 ret = hpre_register_ecdh(qm);
2227 if (ret)
2228 goto unreg_dh;
2229
2230 ret = hpre_register_x25519(qm);
2231 if (ret)
2232 goto unreg_ecdh;
2233
2234 hpre_available_devs++;
2235 mutex_unlock(&hpre_algs_lock);
2236
2237 return ret;
2238
2239 unreg_ecdh:
2240 hpre_unregister_ecdh(qm);
2241 unreg_dh:
2242 hpre_unregister_dh(qm);
2243 unreg_rsa:
2244 hpre_unregister_rsa(qm);
2245 unlock:
2246 mutex_unlock(&hpre_algs_lock);
2247 return ret;
2248 }
2249
hpre_algs_unregister(struct hisi_qm * qm)2250 void hpre_algs_unregister(struct hisi_qm *qm)
2251 {
2252 mutex_lock(&hpre_algs_lock);
2253 if (--hpre_available_devs)
2254 goto unlock;
2255
2256 hpre_unregister_x25519(qm);
2257 hpre_unregister_ecdh(qm);
2258 hpre_unregister_dh(qm);
2259 hpre_unregister_rsa(qm);
2260
2261 unlock:
2262 mutex_unlock(&hpre_algs_lock);
2263 }
2264