1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/akcipher.h> 4 #include <crypto/curve25519.h> 5 #include <crypto/dh.h> 6 #include <crypto/ecc_curve.h> 7 #include <crypto/ecdh.h> 8 #include <crypto/rng.h> 9 #include <crypto/internal/akcipher.h> 10 #include <crypto/internal/kpp.h> 11 #include <crypto/internal/rsa.h> 12 #include <crypto/kpp.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/fips.h> 16 #include <linux/module.h> 17 #include <linux/time.h> 18 #include "hpre.h" 19 20 struct hpre_ctx; 21 22 #define HPRE_CRYPTO_ALG_PRI 1000 23 #define HPRE_ALIGN_SZ 64 24 #define HPRE_BITS_2_BYTES_SHIFT 3 25 #define HPRE_RSA_512BITS_KSZ 64 26 #define HPRE_RSA_1536BITS_KSZ 192 27 #define HPRE_CRT_PRMS 5 28 #define HPRE_CRT_Q 2 29 #define HPRE_CRT_P 3 30 #define HPRE_CRT_INV 4 31 #define HPRE_DH_G_FLAG 0x02 32 #define HPRE_TRY_SEND_TIMES 100 33 #define HPRE_INVLD_REQ_ID (-1) 34 35 #define HPRE_SQE_ALG_BITS 5 36 #define HPRE_SQE_DONE_SHIFT 30 37 #define HPRE_DH_MAX_P_SZ 512 38 39 #define HPRE_DFX_SEC_TO_US 1000000 40 #define HPRE_DFX_US_TO_NS 1000 41 42 /* due to nist p521 */ 43 #define HPRE_ECC_MAX_KSZ 66 44 45 /* size in bytes of the n prime */ 46 #define HPRE_ECC_NIST_P192_N_SIZE 24 47 #define HPRE_ECC_NIST_P256_N_SIZE 32 48 #define HPRE_ECC_NIST_P384_N_SIZE 48 49 50 /* size in bytes */ 51 #define HPRE_ECC_HW256_KSZ_B 32 52 #define HPRE_ECC_HW384_KSZ_B 48 53 54 /* capability register mask of driver */ 55 #define HPRE_DRV_RSA_MASK_CAP BIT(0) 56 #define HPRE_DRV_DH_MASK_CAP BIT(1) 57 #define HPRE_DRV_ECDH_MASK_CAP BIT(2) 58 #define HPRE_DRV_X25519_MASK_CAP BIT(5) 59 60 static DEFINE_MUTEX(hpre_algs_lock); 61 static unsigned int hpre_available_devs; 62 63 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); 64 65 struct hpre_rsa_ctx { 66 /* low address: e--->n */ 67 char *pubkey; 68 dma_addr_t dma_pubkey; 69 70 /* low address: d--->n */ 71 char *prikey; 72 dma_addr_t dma_prikey; 73 74 /* low address: dq->dp->q->p->qinv */ 75 char *crt_prikey; 76 dma_addr_t dma_crt_prikey; 77 78 struct crypto_akcipher *soft_tfm; 79 }; 80 81 struct hpre_dh_ctx { 82 /* 83 * If base is g we compute the public key 84 * ya = g^xa mod p; [RFC2631 sec 2.1.1] 85 * else if base if the counterpart public key we 86 * compute the shared secret 87 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] 88 * low address: d--->n, please refer to Hisilicon HPRE UM 89 */ 90 char *xa_p; 91 dma_addr_t dma_xa_p; 92 93 char *g; /* m */ 94 dma_addr_t dma_g; 95 }; 96 97 struct hpre_ecdh_ctx { 98 /* low address: p->a->k->b */ 99 unsigned char *p; 100 dma_addr_t dma_p; 101 102 /* low address: x->y */ 103 unsigned char *g; 104 dma_addr_t dma_g; 105 }; 106 107 struct hpre_curve25519_ctx { 108 /* low address: p->a->k */ 109 unsigned char *p; 110 dma_addr_t dma_p; 111 112 /* gx coordinate */ 113 unsigned char *g; 114 dma_addr_t dma_g; 115 }; 116 117 struct hpre_ctx { 118 struct hisi_qp *qp; 119 struct device *dev; 120 struct hpre_asym_request **req_list; 121 struct hpre *hpre; 122 spinlock_t req_lock; 123 unsigned int key_sz; 124 bool crt_g2_mode; 125 struct idr req_idr; 126 union { 127 struct hpre_rsa_ctx rsa; 128 struct hpre_dh_ctx dh; 129 struct hpre_ecdh_ctx ecdh; 130 struct hpre_curve25519_ctx curve25519; 131 }; 132 /* for ecc algorithms */ 133 unsigned int curve_id; 134 }; 135 136 struct hpre_asym_request { 137 char *src; 138 char *dst; 139 struct hpre_sqe req; 140 struct hpre_ctx *ctx; 141 union { 142 struct akcipher_request *rsa; 143 struct kpp_request *dh; 144 struct kpp_request *ecdh; 145 struct kpp_request *curve25519; 146 } areq; 147 int err; 148 int req_id; 149 hpre_cb cb; 150 struct timespec64 req_time; 151 }; 152 153 static inline unsigned int hpre_align_sz(void) 154 { 155 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1; 156 } 157 158 static inline unsigned int hpre_align_pd(void) 159 { 160 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); 161 } 162 163 static int hpre_alloc_req_id(struct hpre_ctx *ctx) 164 { 165 unsigned long flags; 166 int id; 167 168 spin_lock_irqsave(&ctx->req_lock, flags); 169 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); 170 spin_unlock_irqrestore(&ctx->req_lock, flags); 171 172 return id; 173 } 174 175 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) 176 { 177 unsigned long flags; 178 179 spin_lock_irqsave(&ctx->req_lock, flags); 180 idr_remove(&ctx->req_idr, req_id); 181 spin_unlock_irqrestore(&ctx->req_lock, flags); 182 } 183 184 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) 185 { 186 struct hpre_ctx *ctx; 187 struct hpre_dfx *dfx; 188 int id; 189 190 ctx = hpre_req->ctx; 191 id = hpre_alloc_req_id(ctx); 192 if (unlikely(id < 0)) 193 return -EINVAL; 194 195 ctx->req_list[id] = hpre_req; 196 hpre_req->req_id = id; 197 198 dfx = ctx->hpre->debug.dfx; 199 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 200 ktime_get_ts64(&hpre_req->req_time); 201 202 return id; 203 } 204 205 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) 206 { 207 struct hpre_ctx *ctx = hpre_req->ctx; 208 int id = hpre_req->req_id; 209 210 if (hpre_req->req_id >= 0) { 211 hpre_req->req_id = HPRE_INVLD_REQ_ID; 212 ctx->req_list[id] = NULL; 213 hpre_free_req_id(ctx, id); 214 } 215 } 216 217 static struct hisi_qp *hpre_get_qp_and_start(u8 type) 218 { 219 struct hisi_qp *qp; 220 int ret; 221 222 qp = hpre_create_qp(type); 223 if (!qp) { 224 pr_err("Can not create hpre qp!\n"); 225 return ERR_PTR(-ENODEV); 226 } 227 228 ret = hisi_qm_start_qp(qp, 0); 229 if (ret < 0) { 230 hisi_qm_free_qps(&qp, 1); 231 pci_err(qp->qm->pdev, "Can not start qp!\n"); 232 return ERR_PTR(-EINVAL); 233 } 234 235 return qp; 236 } 237 238 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, 239 struct scatterlist *data, unsigned int len, 240 int is_src, dma_addr_t *tmp) 241 { 242 struct device *dev = hpre_req->ctx->dev; 243 enum dma_data_direction dma_dir; 244 245 if (is_src) { 246 hpre_req->src = NULL; 247 dma_dir = DMA_TO_DEVICE; 248 } else { 249 hpre_req->dst = NULL; 250 dma_dir = DMA_FROM_DEVICE; 251 } 252 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); 253 if (unlikely(dma_mapping_error(dev, *tmp))) { 254 dev_err(dev, "dma map data err!\n"); 255 return -ENOMEM; 256 } 257 258 return 0; 259 } 260 261 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, 262 struct scatterlist *data, unsigned int len, 263 int is_src, dma_addr_t *tmp) 264 { 265 struct hpre_ctx *ctx = hpre_req->ctx; 266 struct device *dev = ctx->dev; 267 void *ptr; 268 int shift; 269 270 shift = ctx->key_sz - len; 271 if (unlikely(shift < 0)) 272 return -EINVAL; 273 274 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC); 275 if (unlikely(!ptr)) 276 return -ENOMEM; 277 278 if (is_src) { 279 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0); 280 hpre_req->src = ptr; 281 } else { 282 hpre_req->dst = ptr; 283 } 284 285 return 0; 286 } 287 288 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, 289 struct scatterlist *data, unsigned int len, 290 int is_src, int is_dh) 291 { 292 struct hpre_sqe *msg = &hpre_req->req; 293 struct hpre_ctx *ctx = hpre_req->ctx; 294 dma_addr_t tmp = 0; 295 int ret; 296 297 /* when the data is dh's source, we should format it */ 298 if ((sg_is_last(data) && len == ctx->key_sz) && 299 ((is_dh && !is_src) || !is_dh)) 300 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); 301 else 302 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); 303 304 if (unlikely(ret)) 305 return ret; 306 307 if (is_src) 308 msg->in = cpu_to_le64(tmp); 309 else 310 msg->out = cpu_to_le64(tmp); 311 312 return 0; 313 } 314 315 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, 316 struct hpre_asym_request *req, 317 struct scatterlist *dst, 318 struct scatterlist *src) 319 { 320 struct device *dev = ctx->dev; 321 struct hpre_sqe *sqe = &req->req; 322 dma_addr_t tmp; 323 324 tmp = le64_to_cpu(sqe->in); 325 if (unlikely(dma_mapping_error(dev, tmp))) 326 return; 327 328 if (src) { 329 if (req->src) 330 dma_free_coherent(dev, ctx->key_sz, req->src, tmp); 331 else 332 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); 333 } 334 335 tmp = le64_to_cpu(sqe->out); 336 if (unlikely(dma_mapping_error(dev, tmp))) 337 return; 338 339 if (req->dst) { 340 if (dst) 341 scatterwalk_map_and_copy(req->dst, dst, 0, 342 ctx->key_sz, 1); 343 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp); 344 } else { 345 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE); 346 } 347 } 348 349 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, 350 void **kreq) 351 { 352 struct hpre_asym_request *req; 353 unsigned int err, done, alg; 354 int id; 355 356 #define HPRE_NO_HW_ERR 0 357 #define HPRE_HW_TASK_DONE 3 358 #define HREE_HW_ERR_MASK GENMASK(10, 0) 359 #define HREE_SQE_DONE_MASK GENMASK(1, 0) 360 #define HREE_ALG_TYPE_MASK GENMASK(4, 0) 361 id = (int)le16_to_cpu(sqe->tag); 362 req = ctx->req_list[id]; 363 hpre_rm_req_from_ctx(req); 364 *kreq = req; 365 366 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & 367 HREE_HW_ERR_MASK; 368 369 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & 370 HREE_SQE_DONE_MASK; 371 372 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) 373 return 0; 374 375 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; 376 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", 377 alg, done, err); 378 379 return -EINVAL; 380 } 381 382 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) 383 { 384 struct hpre *hpre; 385 386 if (!ctx || !qp || qlen < 0) 387 return -EINVAL; 388 389 spin_lock_init(&ctx->req_lock); 390 ctx->qp = qp; 391 ctx->dev = &qp->qm->pdev->dev; 392 393 hpre = container_of(ctx->qp->qm, struct hpre, qm); 394 ctx->hpre = hpre; 395 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); 396 if (!ctx->req_list) 397 return -ENOMEM; 398 ctx->key_sz = 0; 399 ctx->crt_g2_mode = false; 400 idr_init(&ctx->req_idr); 401 402 return 0; 403 } 404 405 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) 406 { 407 if (is_clear_all) { 408 idr_destroy(&ctx->req_idr); 409 kfree(ctx->req_list); 410 hisi_qm_free_qps(&ctx->qp, 1); 411 } 412 413 ctx->crt_g2_mode = false; 414 ctx->key_sz = 0; 415 } 416 417 static bool hpre_is_bd_timeout(struct hpre_asym_request *req, 418 u64 overtime_thrhld) 419 { 420 struct timespec64 reply_time; 421 u64 time_use_us; 422 423 ktime_get_ts64(&reply_time); 424 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) * 425 HPRE_DFX_SEC_TO_US + 426 (reply_time.tv_nsec - req->req_time.tv_nsec) / 427 HPRE_DFX_US_TO_NS; 428 429 if (time_use_us <= overtime_thrhld) 430 return false; 431 432 return true; 433 } 434 435 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp) 436 { 437 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 438 struct hpre_asym_request *req; 439 struct kpp_request *areq; 440 u64 overtime_thrhld; 441 int ret; 442 443 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 444 areq = req->areq.dh; 445 areq->dst_len = ctx->key_sz; 446 447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 450 451 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 452 kpp_request_complete(areq, ret); 453 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 454 } 455 456 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) 457 { 458 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 459 struct hpre_asym_request *req; 460 struct akcipher_request *areq; 461 u64 overtime_thrhld; 462 int ret; 463 464 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 465 466 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 467 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 468 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 469 470 areq = req->areq.rsa; 471 areq->dst_len = ctx->key_sz; 472 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 473 akcipher_request_complete(areq, ret); 474 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 475 } 476 477 static void hpre_alg_cb(struct hisi_qp *qp, void *resp) 478 { 479 struct hpre_ctx *ctx = qp->qp_ctx; 480 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 481 struct hpre_sqe *sqe = resp; 482 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; 483 484 if (unlikely(!req)) { 485 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); 486 return; 487 } 488 489 req->cb(ctx, resp); 490 } 491 492 static void hpre_stop_qp_and_put(struct hisi_qp *qp) 493 { 494 hisi_qm_stop_qp(qp); 495 hisi_qm_free_qps(&qp, 1); 496 } 497 498 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) 499 { 500 struct hisi_qp *qp; 501 int ret; 502 503 qp = hpre_get_qp_and_start(type); 504 if (IS_ERR(qp)) 505 return PTR_ERR(qp); 506 507 qp->qp_ctx = ctx; 508 qp->req_cb = hpre_alg_cb; 509 510 ret = hpre_ctx_set(ctx, qp, qp->sq_depth); 511 if (ret) 512 hpre_stop_qp_and_put(qp); 513 514 return ret; 515 } 516 517 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) 518 { 519 struct hpre_asym_request *h_req; 520 struct hpre_sqe *msg; 521 int req_id; 522 void *tmp; 523 524 if (is_rsa) { 525 struct akcipher_request *akreq = req; 526 527 if (akreq->dst_len < ctx->key_sz) { 528 akreq->dst_len = ctx->key_sz; 529 return -EOVERFLOW; 530 } 531 532 tmp = akcipher_request_ctx(akreq); 533 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 534 h_req->cb = hpre_rsa_cb; 535 h_req->areq.rsa = akreq; 536 msg = &h_req->req; 537 memset(msg, 0, sizeof(*msg)); 538 } else { 539 struct kpp_request *kreq = req; 540 541 if (kreq->dst_len < ctx->key_sz) { 542 kreq->dst_len = ctx->key_sz; 543 return -EOVERFLOW; 544 } 545 546 tmp = kpp_request_ctx(kreq); 547 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 548 h_req->cb = hpre_dh_cb; 549 h_req->areq.dh = kreq; 550 msg = &h_req->req; 551 memset(msg, 0, sizeof(*msg)); 552 msg->key = cpu_to_le64(ctx->dh.dma_xa_p); 553 } 554 555 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 556 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 557 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); 558 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 559 h_req->ctx = ctx; 560 561 req_id = hpre_add_req_to_ctx(h_req); 562 if (req_id < 0) 563 return -EBUSY; 564 565 msg->tag = cpu_to_le16((u16)req_id); 566 567 return 0; 568 } 569 570 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) 571 { 572 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 573 int ctr = 0; 574 int ret; 575 576 do { 577 atomic64_inc(&dfx[HPRE_SEND_CNT].value); 578 ret = hisi_qp_send(ctx->qp, msg); 579 if (ret != -EBUSY) 580 break; 581 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); 582 } while (ctr++ < HPRE_TRY_SEND_TIMES); 583 584 if (likely(!ret)) 585 return ret; 586 587 if (ret != -EBUSY) 588 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value); 589 590 return ret; 591 } 592 593 static int hpre_dh_compute_value(struct kpp_request *req) 594 { 595 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 596 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 597 void *tmp = kpp_request_ctx(req); 598 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 599 struct hpre_sqe *msg = &hpre_req->req; 600 int ret; 601 602 ret = hpre_msg_request_set(ctx, req, false); 603 if (unlikely(ret)) 604 return ret; 605 606 if (req->src) { 607 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); 608 if (unlikely(ret)) 609 goto clear_all; 610 } else { 611 msg->in = cpu_to_le64(ctx->dh.dma_g); 612 } 613 614 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); 615 if (unlikely(ret)) 616 goto clear_all; 617 618 if (ctx->crt_g2_mode && !req->src) 619 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2); 620 else 621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH); 622 623 /* success */ 624 ret = hpre_send(ctx, msg); 625 if (likely(!ret)) 626 return -EINPROGRESS; 627 628 clear_all: 629 hpre_rm_req_from_ctx(hpre_req); 630 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 631 632 return ret; 633 } 634 635 static int hpre_is_dh_params_length_valid(unsigned int key_sz) 636 { 637 #define _HPRE_DH_GRP1 768 638 #define _HPRE_DH_GRP2 1024 639 #define _HPRE_DH_GRP5 1536 640 #define _HPRE_DH_GRP14 2048 641 #define _HPRE_DH_GRP15 3072 642 #define _HPRE_DH_GRP16 4096 643 switch (key_sz) { 644 case _HPRE_DH_GRP1: 645 case _HPRE_DH_GRP2: 646 case _HPRE_DH_GRP5: 647 case _HPRE_DH_GRP14: 648 case _HPRE_DH_GRP15: 649 case _HPRE_DH_GRP16: 650 return 0; 651 default: 652 return -EINVAL; 653 } 654 } 655 656 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) 657 { 658 struct device *dev = ctx->dev; 659 unsigned int sz; 660 661 if (params->p_size > HPRE_DH_MAX_P_SZ) 662 return -EINVAL; 663 664 if (hpre_is_dh_params_length_valid(params->p_size << 665 HPRE_BITS_2_BYTES_SHIFT)) 666 return -EINVAL; 667 668 sz = ctx->key_sz = params->p_size; 669 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, 670 &ctx->dh.dma_xa_p, GFP_KERNEL); 671 if (!ctx->dh.xa_p) 672 return -ENOMEM; 673 674 memcpy(ctx->dh.xa_p + sz, params->p, sz); 675 676 /* If g equals 2 don't copy it */ 677 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) { 678 ctx->crt_g2_mode = true; 679 return 0; 680 } 681 682 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL); 683 if (!ctx->dh.g) { 684 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 685 ctx->dh.dma_xa_p); 686 ctx->dh.xa_p = NULL; 687 return -ENOMEM; 688 } 689 690 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size); 691 692 return 0; 693 } 694 695 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 696 { 697 struct device *dev = ctx->dev; 698 unsigned int sz = ctx->key_sz; 699 700 if (is_clear_all) 701 hisi_qm_stop_qp(ctx->qp); 702 703 if (ctx->dh.g) { 704 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); 705 ctx->dh.g = NULL; 706 } 707 708 if (ctx->dh.xa_p) { 709 memzero_explicit(ctx->dh.xa_p, sz); 710 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 711 ctx->dh.dma_xa_p); 712 ctx->dh.xa_p = NULL; 713 } 714 715 hpre_ctx_clear(ctx, is_clear_all); 716 } 717 718 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, 719 unsigned int len) 720 { 721 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 722 struct dh params; 723 int ret; 724 725 if (crypto_dh_decode_key(buf, len, ¶ms) < 0) 726 return -EINVAL; 727 728 /* Free old secret if any */ 729 hpre_dh_clear_ctx(ctx, false); 730 731 ret = hpre_dh_set_params(ctx, ¶ms); 732 if (ret < 0) 733 goto err_clear_ctx; 734 735 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, 736 params.key_size); 737 738 return 0; 739 740 err_clear_ctx: 741 hpre_dh_clear_ctx(ctx, false); 742 return ret; 743 } 744 745 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) 746 { 747 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 748 749 return ctx->key_sz; 750 } 751 752 static int hpre_dh_init_tfm(struct crypto_kpp *tfm) 753 { 754 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 755 756 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 757 758 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 759 } 760 761 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) 762 { 763 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 764 765 hpre_dh_clear_ctx(ctx, true); 766 } 767 768 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) 769 { 770 while (!**ptr && *len) { 771 (*ptr)++; 772 (*len)--; 773 } 774 } 775 776 static bool hpre_rsa_key_size_is_support(unsigned int len) 777 { 778 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT; 779 780 #define _RSA_1024BITS_KEY_WDTH 1024 781 #define _RSA_2048BITS_KEY_WDTH 2048 782 #define _RSA_3072BITS_KEY_WDTH 3072 783 #define _RSA_4096BITS_KEY_WDTH 4096 784 785 switch (bits) { 786 case _RSA_1024BITS_KEY_WDTH: 787 case _RSA_2048BITS_KEY_WDTH: 788 case _RSA_3072BITS_KEY_WDTH: 789 case _RSA_4096BITS_KEY_WDTH: 790 return true; 791 default: 792 return false; 793 } 794 } 795 796 static int hpre_rsa_enc(struct akcipher_request *req) 797 { 798 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 799 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 800 void *tmp = akcipher_request_ctx(req); 801 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 802 struct hpre_sqe *msg = &hpre_req->req; 803 int ret; 804 805 /* For 512 and 1536 bits key size, use soft tfm instead */ 806 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 807 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 808 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 809 ret = crypto_akcipher_encrypt(req); 810 akcipher_request_set_tfm(req, tfm); 811 return ret; 812 } 813 814 if (unlikely(!ctx->rsa.pubkey)) 815 return -EINVAL; 816 817 ret = hpre_msg_request_set(ctx, req, true); 818 if (unlikely(ret)) 819 return ret; 820 821 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); 822 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); 823 824 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 825 if (unlikely(ret)) 826 goto clear_all; 827 828 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 829 if (unlikely(ret)) 830 goto clear_all; 831 832 /* success */ 833 ret = hpre_send(ctx, msg); 834 if (likely(!ret)) 835 return -EINPROGRESS; 836 837 clear_all: 838 hpre_rm_req_from_ctx(hpre_req); 839 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 840 841 return ret; 842 } 843 844 static int hpre_rsa_dec(struct akcipher_request *req) 845 { 846 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 847 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 848 void *tmp = akcipher_request_ctx(req); 849 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 850 struct hpre_sqe *msg = &hpre_req->req; 851 int ret; 852 853 /* For 512 and 1536 bits key size, use soft tfm instead */ 854 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 855 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 856 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 857 ret = crypto_akcipher_decrypt(req); 858 akcipher_request_set_tfm(req, tfm); 859 return ret; 860 } 861 862 if (unlikely(!ctx->rsa.prikey)) 863 return -EINVAL; 864 865 ret = hpre_msg_request_set(ctx, req, true); 866 if (unlikely(ret)) 867 return ret; 868 869 if (ctx->crt_g2_mode) { 870 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); 871 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 872 HPRE_ALG_NC_CRT); 873 } else { 874 msg->key = cpu_to_le64(ctx->rsa.dma_prikey); 875 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 876 HPRE_ALG_NC_NCRT); 877 } 878 879 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 880 if (unlikely(ret)) 881 goto clear_all; 882 883 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 884 if (unlikely(ret)) 885 goto clear_all; 886 887 /* success */ 888 ret = hpre_send(ctx, msg); 889 if (likely(!ret)) 890 return -EINPROGRESS; 891 892 clear_all: 893 hpre_rm_req_from_ctx(hpre_req); 894 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 895 896 return ret; 897 } 898 899 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, 900 size_t vlen, bool private) 901 { 902 const char *ptr = value; 903 904 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 905 906 ctx->key_sz = vlen; 907 908 /* if invalid key size provided, we use software tfm */ 909 if (!hpre_rsa_key_size_is_support(ctx->key_sz)) 910 return 0; 911 912 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, 913 &ctx->rsa.dma_pubkey, 914 GFP_KERNEL); 915 if (!ctx->rsa.pubkey) 916 return -ENOMEM; 917 918 if (private) { 919 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, 920 &ctx->rsa.dma_prikey, 921 GFP_KERNEL); 922 if (!ctx->rsa.prikey) { 923 dma_free_coherent(ctx->dev, vlen << 1, 924 ctx->rsa.pubkey, 925 ctx->rsa.dma_pubkey); 926 ctx->rsa.pubkey = NULL; 927 return -ENOMEM; 928 } 929 memcpy(ctx->rsa.prikey + vlen, ptr, vlen); 930 } 931 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen); 932 933 /* Using hardware HPRE to do RSA */ 934 return 1; 935 } 936 937 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value, 938 size_t vlen) 939 { 940 const char *ptr = value; 941 942 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 943 944 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 945 return -EINVAL; 946 947 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen); 948 949 return 0; 950 } 951 952 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value, 953 size_t vlen) 954 { 955 const char *ptr = value; 956 957 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 958 959 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 960 return -EINVAL; 961 962 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen); 963 964 return 0; 965 } 966 967 static int hpre_crt_para_get(char *para, size_t para_sz, 968 const char *raw, size_t raw_sz) 969 { 970 const char *ptr = raw; 971 size_t len = raw_sz; 972 973 hpre_rsa_drop_leading_zeros(&ptr, &len); 974 if (!len || len > para_sz) 975 return -EINVAL; 976 977 memcpy(para + para_sz - len, ptr, len); 978 979 return 0; 980 } 981 982 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) 983 { 984 unsigned int hlf_ksz = ctx->key_sz >> 1; 985 struct device *dev = ctx->dev; 986 u64 offset; 987 int ret; 988 989 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, 990 &ctx->rsa.dma_crt_prikey, 991 GFP_KERNEL); 992 if (!ctx->rsa.crt_prikey) 993 return -ENOMEM; 994 995 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz, 996 rsa_key->dq, rsa_key->dq_sz); 997 if (ret) 998 goto free_key; 999 1000 offset = hlf_ksz; 1001 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1002 rsa_key->dp, rsa_key->dp_sz); 1003 if (ret) 1004 goto free_key; 1005 1006 offset = hlf_ksz * HPRE_CRT_Q; 1007 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1008 rsa_key->q, rsa_key->q_sz); 1009 if (ret) 1010 goto free_key; 1011 1012 offset = hlf_ksz * HPRE_CRT_P; 1013 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1014 rsa_key->p, rsa_key->p_sz); 1015 if (ret) 1016 goto free_key; 1017 1018 offset = hlf_ksz * HPRE_CRT_INV; 1019 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1020 rsa_key->qinv, rsa_key->qinv_sz); 1021 if (ret) 1022 goto free_key; 1023 1024 ctx->crt_g2_mode = true; 1025 1026 return 0; 1027 1028 free_key: 1029 offset = hlf_ksz * HPRE_CRT_PRMS; 1030 memzero_explicit(ctx->rsa.crt_prikey, offset); 1031 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey, 1032 ctx->rsa.dma_crt_prikey); 1033 ctx->rsa.crt_prikey = NULL; 1034 ctx->crt_g2_mode = false; 1035 1036 return ret; 1037 } 1038 1039 /* If it is clear all, all the resources of the QP will be cleaned. */ 1040 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 1041 { 1042 unsigned int half_key_sz = ctx->key_sz >> 1; 1043 struct device *dev = ctx->dev; 1044 1045 if (is_clear_all) 1046 hisi_qm_stop_qp(ctx->qp); 1047 1048 if (ctx->rsa.pubkey) { 1049 dma_free_coherent(dev, ctx->key_sz << 1, 1050 ctx->rsa.pubkey, ctx->rsa.dma_pubkey); 1051 ctx->rsa.pubkey = NULL; 1052 } 1053 1054 if (ctx->rsa.crt_prikey) { 1055 memzero_explicit(ctx->rsa.crt_prikey, 1056 half_key_sz * HPRE_CRT_PRMS); 1057 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS, 1058 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey); 1059 ctx->rsa.crt_prikey = NULL; 1060 } 1061 1062 if (ctx->rsa.prikey) { 1063 memzero_explicit(ctx->rsa.prikey, ctx->key_sz); 1064 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey, 1065 ctx->rsa.dma_prikey); 1066 ctx->rsa.prikey = NULL; 1067 } 1068 1069 hpre_ctx_clear(ctx, is_clear_all); 1070 } 1071 1072 /* 1073 * we should judge if it is CRT or not, 1074 * CRT: return true, N-CRT: return false . 1075 */ 1076 static bool hpre_is_crt_key(struct rsa_key *key) 1077 { 1078 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz + 1079 key->qinv_sz; 1080 1081 #define LEN_OF_NCRT_PARA 5 1082 1083 /* N-CRT less than 5 parameters */ 1084 return len > LEN_OF_NCRT_PARA; 1085 } 1086 1087 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, 1088 unsigned int keylen, bool private) 1089 { 1090 struct rsa_key rsa_key; 1091 int ret; 1092 1093 hpre_rsa_clear_ctx(ctx, false); 1094 1095 if (private) 1096 ret = rsa_parse_priv_key(&rsa_key, key, keylen); 1097 else 1098 ret = rsa_parse_pub_key(&rsa_key, key, keylen); 1099 if (ret < 0) 1100 return ret; 1101 1102 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private); 1103 if (ret <= 0) 1104 return ret; 1105 1106 if (private) { 1107 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); 1108 if (ret < 0) 1109 goto free; 1110 1111 if (hpre_is_crt_key(&rsa_key)) { 1112 ret = hpre_rsa_setkey_crt(ctx, &rsa_key); 1113 if (ret < 0) 1114 goto free; 1115 } 1116 } 1117 1118 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); 1119 if (ret < 0) 1120 goto free; 1121 1122 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) { 1123 ret = -EINVAL; 1124 goto free; 1125 } 1126 1127 return 0; 1128 1129 free: 1130 hpre_rsa_clear_ctx(ctx, false); 1131 return ret; 1132 } 1133 1134 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, 1135 unsigned int keylen) 1136 { 1137 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1138 int ret; 1139 1140 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen); 1141 if (ret) 1142 return ret; 1143 1144 return hpre_rsa_setkey(ctx, key, keylen, false); 1145 } 1146 1147 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, 1148 unsigned int keylen) 1149 { 1150 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1151 int ret; 1152 1153 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen); 1154 if (ret) 1155 return ret; 1156 1157 return hpre_rsa_setkey(ctx, key, keylen, true); 1158 } 1159 1160 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) 1161 { 1162 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1163 1164 /* For 512 and 1536 bits key size, use soft tfm instead */ 1165 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 1166 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) 1167 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); 1168 1169 return ctx->key_sz; 1170 } 1171 1172 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) 1173 { 1174 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1175 int ret; 1176 1177 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0); 1178 if (IS_ERR(ctx->rsa.soft_tfm)) { 1179 pr_err("Can not alloc_akcipher!\n"); 1180 return PTR_ERR(ctx->rsa.soft_tfm); 1181 } 1182 1183 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) + 1184 hpre_align_pd()); 1185 1186 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 1187 if (ret) 1188 crypto_free_akcipher(ctx->rsa.soft_tfm); 1189 1190 return ret; 1191 } 1192 1193 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) 1194 { 1195 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1196 1197 hpre_rsa_clear_ctx(ctx, true); 1198 crypto_free_akcipher(ctx->rsa.soft_tfm); 1199 } 1200 1201 static void hpre_key_to_big_end(u8 *data, int len) 1202 { 1203 int i, j; 1204 1205 for (i = 0; i < len / 2; i++) { 1206 j = len - i - 1; 1207 swap(data[j], data[i]); 1208 } 1209 } 1210 1211 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, 1212 bool is_ecdh) 1213 { 1214 struct device *dev = ctx->dev; 1215 unsigned int sz = ctx->key_sz; 1216 unsigned int shift = sz << 1; 1217 1218 if (is_clear_all) 1219 hisi_qm_stop_qp(ctx->qp); 1220 1221 if (is_ecdh && ctx->ecdh.p) { 1222 /* ecdh: p->a->k->b */ 1223 memzero_explicit(ctx->ecdh.p + shift, sz); 1224 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1225 ctx->ecdh.p = NULL; 1226 } else if (!is_ecdh && ctx->curve25519.p) { 1227 /* curve25519: p->a->k */ 1228 memzero_explicit(ctx->curve25519.p + shift, sz); 1229 dma_free_coherent(dev, sz << 2, ctx->curve25519.p, 1230 ctx->curve25519.dma_p); 1231 ctx->curve25519.p = NULL; 1232 } 1233 1234 hpre_ctx_clear(ctx, is_clear_all); 1235 } 1236 1237 /* 1238 * The bits of 192/224/256/384/521 are supported by HPRE, 1239 * and convert the bits like: 1240 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576; 1241 * If the parameter bit width is insufficient, then we fill in the 1242 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8; 1243 */ 1244 static unsigned int hpre_ecdh_supported_curve(unsigned short id) 1245 { 1246 switch (id) { 1247 case ECC_CURVE_NIST_P192: 1248 case ECC_CURVE_NIST_P256: 1249 return HPRE_ECC_HW256_KSZ_B; 1250 case ECC_CURVE_NIST_P384: 1251 return HPRE_ECC_HW384_KSZ_B; 1252 default: 1253 break; 1254 } 1255 1256 return 0; 1257 } 1258 1259 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits) 1260 { 1261 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64); 1262 u8 i = 0; 1263 1264 while (i < ndigits - 1) { 1265 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64)); 1266 i++; 1267 } 1268 1269 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz); 1270 hpre_key_to_big_end((u8 *)addr, cur_sz); 1271 } 1272 1273 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params, 1274 unsigned int cur_sz) 1275 { 1276 unsigned int shifta = ctx->key_sz << 1; 1277 unsigned int shiftb = ctx->key_sz << 2; 1278 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz; 1279 void *a = ctx->ecdh.p + shifta - cur_sz; 1280 void *b = ctx->ecdh.p + shiftb - cur_sz; 1281 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz; 1282 void *y = ctx->ecdh.g + shifta - cur_sz; 1283 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id); 1284 char *n; 1285 1286 if (unlikely(!curve)) 1287 return -EINVAL; 1288 1289 n = kzalloc(ctx->key_sz, GFP_KERNEL); 1290 if (!n) 1291 return -ENOMEM; 1292 1293 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits); 1294 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits); 1295 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits); 1296 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits); 1297 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits); 1298 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits); 1299 1300 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) { 1301 kfree(n); 1302 return -EINVAL; 1303 } 1304 1305 kfree(n); 1306 return 0; 1307 } 1308 1309 static unsigned int hpre_ecdh_get_curvesz(unsigned short id) 1310 { 1311 switch (id) { 1312 case ECC_CURVE_NIST_P192: 1313 return HPRE_ECC_NIST_P192_N_SIZE; 1314 case ECC_CURVE_NIST_P256: 1315 return HPRE_ECC_NIST_P256_N_SIZE; 1316 case ECC_CURVE_NIST_P384: 1317 return HPRE_ECC_NIST_P384_N_SIZE; 1318 default: 1319 break; 1320 } 1321 1322 return 0; 1323 } 1324 1325 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) 1326 { 1327 struct device *dev = ctx->dev; 1328 unsigned int sz, shift, curve_sz; 1329 int ret; 1330 1331 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id); 1332 if (!ctx->key_sz) 1333 return -EINVAL; 1334 1335 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1336 if (!curve_sz || params->key_size > curve_sz) 1337 return -EINVAL; 1338 1339 sz = ctx->key_sz; 1340 1341 if (!ctx->ecdh.p) { 1342 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p, 1343 GFP_KERNEL); 1344 if (!ctx->ecdh.p) 1345 return -ENOMEM; 1346 } 1347 1348 shift = sz << 2; 1349 ctx->ecdh.g = ctx->ecdh.p + shift; 1350 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift; 1351 1352 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz); 1353 if (ret) { 1354 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret); 1355 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1356 ctx->ecdh.p = NULL; 1357 return ret; 1358 } 1359 1360 return 0; 1361 } 1362 1363 static bool hpre_key_is_zero(char *key, unsigned short key_sz) 1364 { 1365 int i; 1366 1367 for (i = 0; i < key_sz; i++) 1368 if (key[i]) 1369 return false; 1370 1371 return true; 1372 } 1373 1374 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) 1375 { 1376 struct device *dev = ctx->dev; 1377 int ret; 1378 1379 ret = crypto_get_default_rng(); 1380 if (ret) { 1381 dev_err(dev, "failed to get default rng, ret = %d!\n", ret); 1382 return ret; 1383 } 1384 1385 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key, 1386 params->key_size); 1387 crypto_put_default_rng(); 1388 if (ret) 1389 dev_err(dev, "failed to get rng, ret = %d!\n", ret); 1390 1391 return ret; 1392 } 1393 1394 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, 1395 unsigned int len) 1396 { 1397 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1398 unsigned int sz, sz_shift, curve_sz; 1399 struct device *dev = ctx->dev; 1400 char key[HPRE_ECC_MAX_KSZ]; 1401 struct ecdh params; 1402 int ret; 1403 1404 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { 1405 dev_err(dev, "failed to decode ecdh key!\n"); 1406 return -EINVAL; 1407 } 1408 1409 /* Use stdrng to generate private key */ 1410 if (!params.key || !params.key_size) { 1411 params.key = key; 1412 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1413 if (!curve_sz) { 1414 dev_err(dev, "Invalid curve size!\n"); 1415 return -EINVAL; 1416 } 1417 1418 params.key_size = curve_sz - 1; 1419 ret = ecdh_gen_privkey(ctx, ¶ms); 1420 if (ret) 1421 return ret; 1422 } 1423 1424 if (hpre_key_is_zero(params.key, params.key_size)) { 1425 dev_err(dev, "Invalid hpre key!\n"); 1426 return -EINVAL; 1427 } 1428 1429 hpre_ecc_clear_ctx(ctx, false, true); 1430 1431 ret = hpre_ecdh_set_param(ctx, ¶ms); 1432 if (ret < 0) { 1433 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret); 1434 return ret; 1435 } 1436 1437 sz = ctx->key_sz; 1438 sz_shift = (sz << 1) + sz - params.key_size; 1439 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size); 1440 1441 return 0; 1442 } 1443 1444 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, 1445 struct hpre_asym_request *req, 1446 struct scatterlist *dst, 1447 struct scatterlist *src) 1448 { 1449 struct device *dev = ctx->dev; 1450 struct hpre_sqe *sqe = &req->req; 1451 dma_addr_t dma; 1452 1453 dma = le64_to_cpu(sqe->in); 1454 if (unlikely(dma_mapping_error(dev, dma))) 1455 return; 1456 1457 if (src && req->src) 1458 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); 1459 1460 dma = le64_to_cpu(sqe->out); 1461 if (unlikely(dma_mapping_error(dev, dma))) 1462 return; 1463 1464 if (req->dst) 1465 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); 1466 if (dst) 1467 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE); 1468 } 1469 1470 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) 1471 { 1472 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1473 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1474 struct hpre_asym_request *req = NULL; 1475 struct kpp_request *areq; 1476 u64 overtime_thrhld; 1477 char *p; 1478 int ret; 1479 1480 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1481 areq = req->areq.ecdh; 1482 areq->dst_len = ctx->key_sz << 1; 1483 1484 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1485 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1486 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1487 1488 p = sg_virt(areq->dst); 1489 memmove(p, p + ctx->key_sz - curve_sz, curve_sz); 1490 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); 1491 1492 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1493 kpp_request_complete(areq, ret); 1494 1495 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1496 } 1497 1498 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, 1499 struct kpp_request *req) 1500 { 1501 struct hpre_asym_request *h_req; 1502 struct hpre_sqe *msg; 1503 int req_id; 1504 void *tmp; 1505 1506 if (req->dst_len < ctx->key_sz << 1) { 1507 req->dst_len = ctx->key_sz << 1; 1508 return -EINVAL; 1509 } 1510 1511 tmp = kpp_request_ctx(req); 1512 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 1513 h_req->cb = hpre_ecdh_cb; 1514 h_req->areq.ecdh = req; 1515 msg = &h_req->req; 1516 memset(msg, 0, sizeof(*msg)); 1517 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1518 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1519 msg->key = cpu_to_le64(ctx->ecdh.dma_p); 1520 1521 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1522 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1523 h_req->ctx = ctx; 1524 1525 req_id = hpre_add_req_to_ctx(h_req); 1526 if (req_id < 0) 1527 return -EBUSY; 1528 1529 msg->tag = cpu_to_le16((u16)req_id); 1530 return 0; 1531 } 1532 1533 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, 1534 struct scatterlist *data, unsigned int len) 1535 { 1536 struct hpre_sqe *msg = &hpre_req->req; 1537 struct hpre_ctx *ctx = hpre_req->ctx; 1538 struct device *dev = ctx->dev; 1539 unsigned int tmpshift; 1540 dma_addr_t dma = 0; 1541 void *ptr; 1542 int shift; 1543 1544 /* Src_data include gx and gy. */ 1545 shift = ctx->key_sz - (len >> 1); 1546 if (unlikely(shift < 0)) 1547 return -EINVAL; 1548 1549 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL); 1550 if (unlikely(!ptr)) 1551 return -ENOMEM; 1552 1553 tmpshift = ctx->key_sz << 1; 1554 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0); 1555 memcpy(ptr + shift, ptr + tmpshift, len >> 1); 1556 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1); 1557 1558 hpre_req->src = ptr; 1559 msg->in = cpu_to_le64(dma); 1560 return 0; 1561 } 1562 1563 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, 1564 struct scatterlist *data, unsigned int len) 1565 { 1566 struct hpre_sqe *msg = &hpre_req->req; 1567 struct hpre_ctx *ctx = hpre_req->ctx; 1568 struct device *dev = ctx->dev; 1569 dma_addr_t dma; 1570 1571 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { 1572 dev_err(dev, "data or data length is illegal!\n"); 1573 return -EINVAL; 1574 } 1575 1576 hpre_req->dst = NULL; 1577 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1578 if (unlikely(dma_mapping_error(dev, dma))) { 1579 dev_err(dev, "dma map data err!\n"); 1580 return -ENOMEM; 1581 } 1582 1583 msg->out = cpu_to_le64(dma); 1584 return 0; 1585 } 1586 1587 static int hpre_ecdh_compute_value(struct kpp_request *req) 1588 { 1589 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1590 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1591 struct device *dev = ctx->dev; 1592 void *tmp = kpp_request_ctx(req); 1593 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 1594 struct hpre_sqe *msg = &hpre_req->req; 1595 int ret; 1596 1597 ret = hpre_ecdh_msg_request_set(ctx, req); 1598 if (unlikely(ret)) { 1599 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret); 1600 return ret; 1601 } 1602 1603 if (req->src) { 1604 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len); 1605 if (unlikely(ret)) { 1606 dev_err(dev, "failed to init src data, ret = %d!\n", ret); 1607 goto clear_all; 1608 } 1609 } else { 1610 msg->in = cpu_to_le64(ctx->ecdh.dma_g); 1611 } 1612 1613 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len); 1614 if (unlikely(ret)) { 1615 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1616 goto clear_all; 1617 } 1618 1619 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); 1620 ret = hpre_send(ctx, msg); 1621 if (likely(!ret)) 1622 return -EINPROGRESS; 1623 1624 clear_all: 1625 hpre_rm_req_from_ctx(hpre_req); 1626 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1627 return ret; 1628 } 1629 1630 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) 1631 { 1632 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1633 1634 /* max size is the pub_key_size, include x and y */ 1635 return ctx->key_sz << 1; 1636 } 1637 1638 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) 1639 { 1640 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1641 1642 ctx->curve_id = ECC_CURVE_NIST_P192; 1643 1644 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1645 1646 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1647 } 1648 1649 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) 1650 { 1651 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1652 1653 ctx->curve_id = ECC_CURVE_NIST_P256; 1654 1655 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1656 1657 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1658 } 1659 1660 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) 1661 { 1662 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1663 1664 ctx->curve_id = ECC_CURVE_NIST_P384; 1665 1666 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1667 1668 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1669 } 1670 1671 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) 1672 { 1673 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1674 1675 hpre_ecc_clear_ctx(ctx, true, true); 1676 } 1677 1678 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, 1679 unsigned int len) 1680 { 1681 u8 secret[CURVE25519_KEY_SIZE] = { 0 }; 1682 unsigned int sz = ctx->key_sz; 1683 const struct ecc_curve *curve; 1684 unsigned int shift = sz << 1; 1685 void *p; 1686 1687 /* 1688 * The key from 'buf' is in little-endian, we should preprocess it as 1689 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64", 1690 * then convert it to big endian. Only in this way, the result can be 1691 * the same as the software curve-25519 that exists in crypto. 1692 */ 1693 memcpy(secret, buf, len); 1694 curve25519_clamp_secret(secret); 1695 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE); 1696 1697 p = ctx->curve25519.p + sz - len; 1698 1699 curve = ecc_get_curve25519(); 1700 1701 /* fill curve parameters */ 1702 fill_curve_param(p, curve->p, len, curve->g.ndigits); 1703 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits); 1704 memcpy(p + shift, secret, len); 1705 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits); 1706 memzero_explicit(secret, CURVE25519_KEY_SIZE); 1707 } 1708 1709 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, 1710 unsigned int len) 1711 { 1712 struct device *dev = ctx->dev; 1713 unsigned int sz = ctx->key_sz; 1714 unsigned int shift = sz << 1; 1715 1716 /* p->a->k->gx */ 1717 if (!ctx->curve25519.p) { 1718 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2, 1719 &ctx->curve25519.dma_p, 1720 GFP_KERNEL); 1721 if (!ctx->curve25519.p) 1722 return -ENOMEM; 1723 } 1724 1725 ctx->curve25519.g = ctx->curve25519.p + shift + sz; 1726 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz; 1727 1728 hpre_curve25519_fill_curve(ctx, buf, len); 1729 1730 return 0; 1731 } 1732 1733 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, 1734 unsigned int len) 1735 { 1736 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1737 struct device *dev = ctx->dev; 1738 int ret = -EINVAL; 1739 1740 if (len != CURVE25519_KEY_SIZE || 1741 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) { 1742 dev_err(dev, "key is null or key len is not 32bytes!\n"); 1743 return ret; 1744 } 1745 1746 /* Free old secret if any */ 1747 hpre_ecc_clear_ctx(ctx, false, false); 1748 1749 ctx->key_sz = CURVE25519_KEY_SIZE; 1750 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE); 1751 if (ret) { 1752 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret); 1753 hpre_ecc_clear_ctx(ctx, false, false); 1754 return ret; 1755 } 1756 1757 return 0; 1758 } 1759 1760 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, 1761 struct hpre_asym_request *req, 1762 struct scatterlist *dst, 1763 struct scatterlist *src) 1764 { 1765 struct device *dev = ctx->dev; 1766 struct hpre_sqe *sqe = &req->req; 1767 dma_addr_t dma; 1768 1769 dma = le64_to_cpu(sqe->in); 1770 if (unlikely(dma_mapping_error(dev, dma))) 1771 return; 1772 1773 if (src && req->src) 1774 dma_free_coherent(dev, ctx->key_sz, req->src, dma); 1775 1776 dma = le64_to_cpu(sqe->out); 1777 if (unlikely(dma_mapping_error(dev, dma))) 1778 return; 1779 1780 if (req->dst) 1781 dma_free_coherent(dev, ctx->key_sz, req->dst, dma); 1782 if (dst) 1783 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE); 1784 } 1785 1786 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) 1787 { 1788 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1789 struct hpre_asym_request *req = NULL; 1790 struct kpp_request *areq; 1791 u64 overtime_thrhld; 1792 int ret; 1793 1794 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1795 areq = req->areq.curve25519; 1796 areq->dst_len = ctx->key_sz; 1797 1798 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1799 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1800 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1801 1802 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); 1803 1804 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1805 kpp_request_complete(areq, ret); 1806 1807 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1808 } 1809 1810 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, 1811 struct kpp_request *req) 1812 { 1813 struct hpre_asym_request *h_req; 1814 struct hpre_sqe *msg; 1815 int req_id; 1816 void *tmp; 1817 1818 if (unlikely(req->dst_len < ctx->key_sz)) { 1819 req->dst_len = ctx->key_sz; 1820 return -EINVAL; 1821 } 1822 1823 tmp = kpp_request_ctx(req); 1824 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 1825 h_req->cb = hpre_curve25519_cb; 1826 h_req->areq.curve25519 = req; 1827 msg = &h_req->req; 1828 memset(msg, 0, sizeof(*msg)); 1829 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1830 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1831 msg->key = cpu_to_le64(ctx->curve25519.dma_p); 1832 1833 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1834 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1835 h_req->ctx = ctx; 1836 1837 req_id = hpre_add_req_to_ctx(h_req); 1838 if (req_id < 0) 1839 return -EBUSY; 1840 1841 msg->tag = cpu_to_le16((u16)req_id); 1842 return 0; 1843 } 1844 1845 static void hpre_curve25519_src_modulo_p(u8 *ptr) 1846 { 1847 int i; 1848 1849 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++) 1850 ptr[i] = 0; 1851 1852 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */ 1853 ptr[i] -= 0xed; 1854 } 1855 1856 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, 1857 struct scatterlist *data, unsigned int len) 1858 { 1859 struct hpre_sqe *msg = &hpre_req->req; 1860 struct hpre_ctx *ctx = hpre_req->ctx; 1861 struct device *dev = ctx->dev; 1862 u8 p[CURVE25519_KEY_SIZE] = { 0 }; 1863 const struct ecc_curve *curve; 1864 dma_addr_t dma = 0; 1865 u8 *ptr; 1866 1867 if (len != CURVE25519_KEY_SIZE) { 1868 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len); 1869 return -EINVAL; 1870 } 1871 1872 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL); 1873 if (unlikely(!ptr)) 1874 return -ENOMEM; 1875 1876 scatterwalk_map_and_copy(ptr, data, 0, len, 0); 1877 1878 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) { 1879 dev_err(dev, "gx is null!\n"); 1880 goto err; 1881 } 1882 1883 /* 1884 * Src_data(gx) is in little-endian order, MSB in the final byte should 1885 * be masked as described in RFC7748, then transform it to big-endian 1886 * form, then hisi_hpre can use the data. 1887 */ 1888 ptr[31] &= 0x7f; 1889 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE); 1890 1891 curve = ecc_get_curve25519(); 1892 1893 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits); 1894 1895 /* 1896 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, 1897 * we get its modulus to p, and then use it. 1898 */ 1899 if (memcmp(ptr, p, ctx->key_sz) == 0) { 1900 dev_err(dev, "gx is p!\n"); 1901 goto err; 1902 } else if (memcmp(ptr, p, ctx->key_sz) > 0) { 1903 hpre_curve25519_src_modulo_p(ptr); 1904 } 1905 1906 hpre_req->src = ptr; 1907 msg->in = cpu_to_le64(dma); 1908 return 0; 1909 1910 err: 1911 dma_free_coherent(dev, ctx->key_sz, ptr, dma); 1912 return -EINVAL; 1913 } 1914 1915 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, 1916 struct scatterlist *data, unsigned int len) 1917 { 1918 struct hpre_sqe *msg = &hpre_req->req; 1919 struct hpre_ctx *ctx = hpre_req->ctx; 1920 struct device *dev = ctx->dev; 1921 dma_addr_t dma; 1922 1923 if (!data || !sg_is_last(data) || len != ctx->key_sz) { 1924 dev_err(dev, "data or data length is illegal!\n"); 1925 return -EINVAL; 1926 } 1927 1928 hpre_req->dst = NULL; 1929 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1930 if (unlikely(dma_mapping_error(dev, dma))) { 1931 dev_err(dev, "dma map data err!\n"); 1932 return -ENOMEM; 1933 } 1934 1935 msg->out = cpu_to_le64(dma); 1936 return 0; 1937 } 1938 1939 static int hpre_curve25519_compute_value(struct kpp_request *req) 1940 { 1941 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1942 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1943 struct device *dev = ctx->dev; 1944 void *tmp = kpp_request_ctx(req); 1945 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 1946 struct hpre_sqe *msg = &hpre_req->req; 1947 int ret; 1948 1949 ret = hpre_curve25519_msg_request_set(ctx, req); 1950 if (unlikely(ret)) { 1951 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret); 1952 return ret; 1953 } 1954 1955 if (req->src) { 1956 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len); 1957 if (unlikely(ret)) { 1958 dev_err(dev, "failed to init src data, ret = %d!\n", 1959 ret); 1960 goto clear_all; 1961 } 1962 } else { 1963 msg->in = cpu_to_le64(ctx->curve25519.dma_g); 1964 } 1965 1966 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len); 1967 if (unlikely(ret)) { 1968 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1969 goto clear_all; 1970 } 1971 1972 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL); 1973 ret = hpre_send(ctx, msg); 1974 if (likely(!ret)) 1975 return -EINPROGRESS; 1976 1977 clear_all: 1978 hpre_rm_req_from_ctx(hpre_req); 1979 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1980 return ret; 1981 } 1982 1983 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm) 1984 { 1985 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1986 1987 return ctx->key_sz; 1988 } 1989 1990 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm) 1991 { 1992 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1993 1994 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1995 1996 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1997 } 1998 1999 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm) 2000 { 2001 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 2002 2003 hpre_ecc_clear_ctx(ctx, true, false); 2004 } 2005 2006 static struct akcipher_alg rsa = { 2007 .sign = hpre_rsa_dec, 2008 .verify = hpre_rsa_enc, 2009 .encrypt = hpre_rsa_enc, 2010 .decrypt = hpre_rsa_dec, 2011 .set_pub_key = hpre_rsa_setpubkey, 2012 .set_priv_key = hpre_rsa_setprivkey, 2013 .max_size = hpre_rsa_max_size, 2014 .init = hpre_rsa_init_tfm, 2015 .exit = hpre_rsa_exit_tfm, 2016 .base = { 2017 .cra_ctxsize = sizeof(struct hpre_ctx), 2018 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2019 .cra_name = "rsa", 2020 .cra_driver_name = "hpre-rsa", 2021 .cra_module = THIS_MODULE, 2022 }, 2023 }; 2024 2025 static struct kpp_alg dh = { 2026 .set_secret = hpre_dh_set_secret, 2027 .generate_public_key = hpre_dh_compute_value, 2028 .compute_shared_secret = hpre_dh_compute_value, 2029 .max_size = hpre_dh_max_size, 2030 .init = hpre_dh_init_tfm, 2031 .exit = hpre_dh_exit_tfm, 2032 .base = { 2033 .cra_ctxsize = sizeof(struct hpre_ctx), 2034 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2035 .cra_name = "dh", 2036 .cra_driver_name = "hpre-dh", 2037 .cra_module = THIS_MODULE, 2038 }, 2039 }; 2040 2041 static struct kpp_alg ecdh_curves[] = { 2042 { 2043 .set_secret = hpre_ecdh_set_secret, 2044 .generate_public_key = hpre_ecdh_compute_value, 2045 .compute_shared_secret = hpre_ecdh_compute_value, 2046 .max_size = hpre_ecdh_max_size, 2047 .init = hpre_ecdh_nist_p192_init_tfm, 2048 .exit = hpre_ecdh_exit_tfm, 2049 .base = { 2050 .cra_ctxsize = sizeof(struct hpre_ctx), 2051 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2052 .cra_name = "ecdh-nist-p192", 2053 .cra_driver_name = "hpre-ecdh-nist-p192", 2054 .cra_module = THIS_MODULE, 2055 }, 2056 }, { 2057 .set_secret = hpre_ecdh_set_secret, 2058 .generate_public_key = hpre_ecdh_compute_value, 2059 .compute_shared_secret = hpre_ecdh_compute_value, 2060 .max_size = hpre_ecdh_max_size, 2061 .init = hpre_ecdh_nist_p256_init_tfm, 2062 .exit = hpre_ecdh_exit_tfm, 2063 .base = { 2064 .cra_ctxsize = sizeof(struct hpre_ctx), 2065 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2066 .cra_name = "ecdh-nist-p256", 2067 .cra_driver_name = "hpre-ecdh-nist-p256", 2068 .cra_module = THIS_MODULE, 2069 }, 2070 }, { 2071 .set_secret = hpre_ecdh_set_secret, 2072 .generate_public_key = hpre_ecdh_compute_value, 2073 .compute_shared_secret = hpre_ecdh_compute_value, 2074 .max_size = hpre_ecdh_max_size, 2075 .init = hpre_ecdh_nist_p384_init_tfm, 2076 .exit = hpre_ecdh_exit_tfm, 2077 .base = { 2078 .cra_ctxsize = sizeof(struct hpre_ctx), 2079 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2080 .cra_name = "ecdh-nist-p384", 2081 .cra_driver_name = "hpre-ecdh-nist-p384", 2082 .cra_module = THIS_MODULE, 2083 }, 2084 } 2085 }; 2086 2087 static struct kpp_alg curve25519_alg = { 2088 .set_secret = hpre_curve25519_set_secret, 2089 .generate_public_key = hpre_curve25519_compute_value, 2090 .compute_shared_secret = hpre_curve25519_compute_value, 2091 .max_size = hpre_curve25519_max_size, 2092 .init = hpre_curve25519_init_tfm, 2093 .exit = hpre_curve25519_exit_tfm, 2094 .base = { 2095 .cra_ctxsize = sizeof(struct hpre_ctx), 2096 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2097 .cra_name = "curve25519", 2098 .cra_driver_name = "hpre-curve25519", 2099 .cra_module = THIS_MODULE, 2100 }, 2101 }; 2102 2103 static int hpre_register_rsa(struct hisi_qm *qm) 2104 { 2105 int ret; 2106 2107 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) 2108 return 0; 2109 2110 rsa.base.cra_flags = 0; 2111 ret = crypto_register_akcipher(&rsa); 2112 if (ret) 2113 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret); 2114 2115 return ret; 2116 } 2117 2118 static void hpre_unregister_rsa(struct hisi_qm *qm) 2119 { 2120 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) 2121 return; 2122 2123 crypto_unregister_akcipher(&rsa); 2124 } 2125 2126 static int hpre_register_dh(struct hisi_qm *qm) 2127 { 2128 int ret; 2129 2130 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) 2131 return 0; 2132 2133 ret = crypto_register_kpp(&dh); 2134 if (ret) 2135 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret); 2136 2137 return ret; 2138 } 2139 2140 static void hpre_unregister_dh(struct hisi_qm *qm) 2141 { 2142 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) 2143 return; 2144 2145 crypto_unregister_kpp(&dh); 2146 } 2147 2148 static int hpre_register_ecdh(struct hisi_qm *qm) 2149 { 2150 int ret, i; 2151 2152 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) 2153 return 0; 2154 2155 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) { 2156 ret = crypto_register_kpp(&ecdh_curves[i]); 2157 if (ret) { 2158 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n", 2159 ecdh_curves[i].base.cra_name, ret); 2160 goto unreg_kpp; 2161 } 2162 } 2163 2164 return 0; 2165 2166 unreg_kpp: 2167 for (--i; i >= 0; --i) 2168 crypto_unregister_kpp(&ecdh_curves[i]); 2169 2170 return ret; 2171 } 2172 2173 static void hpre_unregister_ecdh(struct hisi_qm *qm) 2174 { 2175 int i; 2176 2177 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) 2178 return; 2179 2180 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i) 2181 crypto_unregister_kpp(&ecdh_curves[i]); 2182 } 2183 2184 static int hpre_register_x25519(struct hisi_qm *qm) 2185 { 2186 int ret; 2187 2188 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) 2189 return 0; 2190 2191 ret = crypto_register_kpp(&curve25519_alg); 2192 if (ret) 2193 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret); 2194 2195 return ret; 2196 } 2197 2198 static void hpre_unregister_x25519(struct hisi_qm *qm) 2199 { 2200 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) 2201 return; 2202 2203 crypto_unregister_kpp(&curve25519_alg); 2204 } 2205 2206 int hpre_algs_register(struct hisi_qm *qm) 2207 { 2208 int ret = 0; 2209 2210 mutex_lock(&hpre_algs_lock); 2211 if (hpre_available_devs) { 2212 hpre_available_devs++; 2213 goto unlock; 2214 } 2215 2216 ret = hpre_register_rsa(qm); 2217 if (ret) 2218 goto unlock; 2219 2220 ret = hpre_register_dh(qm); 2221 if (ret) 2222 goto unreg_rsa; 2223 2224 ret = hpre_register_ecdh(qm); 2225 if (ret) 2226 goto unreg_dh; 2227 2228 ret = hpre_register_x25519(qm); 2229 if (ret) 2230 goto unreg_ecdh; 2231 2232 hpre_available_devs++; 2233 mutex_unlock(&hpre_algs_lock); 2234 2235 return ret; 2236 2237 unreg_ecdh: 2238 hpre_unregister_ecdh(qm); 2239 unreg_dh: 2240 hpre_unregister_dh(qm); 2241 unreg_rsa: 2242 hpre_unregister_rsa(qm); 2243 unlock: 2244 mutex_unlock(&hpre_algs_lock); 2245 return ret; 2246 } 2247 2248 void hpre_algs_unregister(struct hisi_qm *qm) 2249 { 2250 mutex_lock(&hpre_algs_lock); 2251 if (--hpre_available_devs) 2252 goto unlock; 2253 2254 hpre_unregister_x25519(qm); 2255 hpre_unregister_ecdh(qm); 2256 hpre_unregister_dh(qm); 2257 hpre_unregister_rsa(qm); 2258 2259 unlock: 2260 mutex_unlock(&hpre_algs_lock); 2261 } 2262