1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/akcipher.h> 4 #include <crypto/curve25519.h> 5 #include <crypto/dh.h> 6 #include <crypto/ecc_curve.h> 7 #include <crypto/ecdh.h> 8 #include <crypto/rng.h> 9 #include <crypto/internal/akcipher.h> 10 #include <crypto/internal/kpp.h> 11 #include <crypto/internal/rsa.h> 12 #include <crypto/kpp.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/fips.h> 16 #include <linux/module.h> 17 #include <linux/time.h> 18 #include "hpre.h" 19 20 struct hpre_ctx; 21 22 #define HPRE_CRYPTO_ALG_PRI 1000 23 #define HPRE_ALIGN_SZ 64 24 #define HPRE_BITS_2_BYTES_SHIFT 3 25 #define HPRE_RSA_512BITS_KSZ 64 26 #define HPRE_RSA_1536BITS_KSZ 192 27 #define HPRE_CRT_PRMS 5 28 #define HPRE_CRT_Q 2 29 #define HPRE_CRT_P 3 30 #define HPRE_CRT_INV 4 31 #define HPRE_DH_G_FLAG 0x02 32 #define HPRE_TRY_SEND_TIMES 100 33 #define HPRE_INVLD_REQ_ID (-1) 34 35 #define HPRE_SQE_ALG_BITS 5 36 #define HPRE_SQE_DONE_SHIFT 30 37 #define HPRE_DH_MAX_P_SZ 512 38 39 #define HPRE_DFX_SEC_TO_US 1000000 40 #define HPRE_DFX_US_TO_NS 1000 41 42 /* due to nist p521 */ 43 #define HPRE_ECC_MAX_KSZ 66 44 45 /* size in bytes of the n prime */ 46 #define HPRE_ECC_NIST_P192_N_SIZE 24 47 #define HPRE_ECC_NIST_P256_N_SIZE 32 48 #define HPRE_ECC_NIST_P384_N_SIZE 48 49 50 /* size in bytes */ 51 #define HPRE_ECC_HW256_KSZ_B 32 52 #define HPRE_ECC_HW384_KSZ_B 48 53 54 /* capability register mask of driver */ 55 #define HPRE_DRV_RSA_MASK_CAP BIT(0) 56 #define HPRE_DRV_DH_MASK_CAP BIT(1) 57 #define HPRE_DRV_ECDH_MASK_CAP BIT(2) 58 #define HPRE_DRV_X25519_MASK_CAP BIT(5) 59 60 static DEFINE_MUTEX(hpre_algs_lock); 61 static unsigned int hpre_available_devs; 62 63 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); 64 65 struct hpre_rsa_ctx { 66 /* low address: e--->n */ 67 char *pubkey; 68 dma_addr_t dma_pubkey; 69 70 /* low address: d--->n */ 71 char *prikey; 72 dma_addr_t dma_prikey; 73 74 /* low address: dq->dp->q->p->qinv */ 75 char *crt_prikey; 76 dma_addr_t dma_crt_prikey; 77 78 struct crypto_akcipher *soft_tfm; 79 }; 80 81 struct hpre_dh_ctx { 82 /* 83 * If base is g we compute the public key 84 * ya = g^xa mod p; [RFC2631 sec 2.1.1] 85 * else if base if the counterpart public key we 86 * compute the shared secret 87 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] 88 * low address: d--->n, please refer to Hisilicon HPRE UM 89 */ 90 char *xa_p; 91 dma_addr_t dma_xa_p; 92 93 char *g; /* m */ 94 dma_addr_t dma_g; 95 }; 96 97 struct hpre_ecdh_ctx { 98 /* low address: p->a->k->b */ 99 unsigned char *p; 100 dma_addr_t dma_p; 101 102 /* low address: x->y */ 103 unsigned char *g; 104 dma_addr_t dma_g; 105 }; 106 107 struct hpre_curve25519_ctx { 108 /* low address: p->a->k */ 109 unsigned char *p; 110 dma_addr_t dma_p; 111 112 /* gx coordinate */ 113 unsigned char *g; 114 dma_addr_t dma_g; 115 }; 116 117 struct hpre_ctx { 118 struct hisi_qp *qp; 119 struct device *dev; 120 struct hpre_asym_request **req_list; 121 struct hpre *hpre; 122 spinlock_t req_lock; 123 unsigned int key_sz; 124 bool crt_g2_mode; 125 struct idr req_idr; 126 union { 127 struct hpre_rsa_ctx rsa; 128 struct hpre_dh_ctx dh; 129 struct hpre_ecdh_ctx ecdh; 130 struct hpre_curve25519_ctx curve25519; 131 }; 132 /* for ecc algorithms */ 133 unsigned int curve_id; 134 }; 135 136 struct hpre_asym_request { 137 char *src; 138 char *dst; 139 struct hpre_sqe req; 140 struct hpre_ctx *ctx; 141 union { 142 struct akcipher_request *rsa; 143 struct kpp_request *dh; 144 struct kpp_request *ecdh; 145 struct kpp_request *curve25519; 146 } areq; 147 int err; 148 int req_id; 149 hpre_cb cb; 150 struct timespec64 req_time; 151 }; 152 153 static inline unsigned int hpre_align_sz(void) 154 { 155 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1; 156 } 157 158 static inline unsigned int hpre_align_pd(void) 159 { 160 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); 161 } 162 163 static int hpre_alloc_req_id(struct hpre_ctx *ctx) 164 { 165 unsigned long flags; 166 int id; 167 168 spin_lock_irqsave(&ctx->req_lock, flags); 169 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); 170 spin_unlock_irqrestore(&ctx->req_lock, flags); 171 172 return id; 173 } 174 175 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) 176 { 177 unsigned long flags; 178 179 spin_lock_irqsave(&ctx->req_lock, flags); 180 idr_remove(&ctx->req_idr, req_id); 181 spin_unlock_irqrestore(&ctx->req_lock, flags); 182 } 183 184 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) 185 { 186 struct hpre_ctx *ctx; 187 struct hpre_dfx *dfx; 188 int id; 189 190 ctx = hpre_req->ctx; 191 id = hpre_alloc_req_id(ctx); 192 if (unlikely(id < 0)) 193 return -EINVAL; 194 195 ctx->req_list[id] = hpre_req; 196 hpre_req->req_id = id; 197 198 dfx = ctx->hpre->debug.dfx; 199 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 200 ktime_get_ts64(&hpre_req->req_time); 201 202 return id; 203 } 204 205 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) 206 { 207 struct hpre_ctx *ctx = hpre_req->ctx; 208 int id = hpre_req->req_id; 209 210 if (hpre_req->req_id >= 0) { 211 hpre_req->req_id = HPRE_INVLD_REQ_ID; 212 ctx->req_list[id] = NULL; 213 hpre_free_req_id(ctx, id); 214 } 215 } 216 217 static struct hisi_qp *hpre_get_qp_and_start(u8 type) 218 { 219 struct hisi_qp *qp; 220 int ret; 221 222 qp = hpre_create_qp(type); 223 if (!qp) { 224 pr_err("Can not create hpre qp!\n"); 225 return ERR_PTR(-ENODEV); 226 } 227 228 ret = hisi_qm_start_qp(qp, 0); 229 if (ret < 0) { 230 hisi_qm_free_qps(&qp, 1); 231 pci_err(qp->qm->pdev, "Can not start qp!\n"); 232 return ERR_PTR(-EINVAL); 233 } 234 235 return qp; 236 } 237 238 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, 239 struct scatterlist *data, unsigned int len, 240 int is_src, dma_addr_t *tmp) 241 { 242 struct device *dev = hpre_req->ctx->dev; 243 enum dma_data_direction dma_dir; 244 245 if (is_src) { 246 hpre_req->src = NULL; 247 dma_dir = DMA_TO_DEVICE; 248 } else { 249 hpre_req->dst = NULL; 250 dma_dir = DMA_FROM_DEVICE; 251 } 252 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); 253 if (unlikely(dma_mapping_error(dev, *tmp))) { 254 dev_err(dev, "dma map data err!\n"); 255 return -ENOMEM; 256 } 257 258 return 0; 259 } 260 261 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, 262 struct scatterlist *data, unsigned int len, 263 int is_src, dma_addr_t *tmp) 264 { 265 struct hpre_ctx *ctx = hpre_req->ctx; 266 struct device *dev = ctx->dev; 267 void *ptr; 268 int shift; 269 270 shift = ctx->key_sz - len; 271 if (unlikely(shift < 0)) 272 return -EINVAL; 273 274 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC); 275 if (unlikely(!ptr)) 276 return -ENOMEM; 277 278 if (is_src) { 279 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0); 280 hpre_req->src = ptr; 281 } else { 282 hpre_req->dst = ptr; 283 } 284 285 return 0; 286 } 287 288 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, 289 struct scatterlist *data, unsigned int len, 290 int is_src, int is_dh) 291 { 292 struct hpre_sqe *msg = &hpre_req->req; 293 struct hpre_ctx *ctx = hpre_req->ctx; 294 dma_addr_t tmp = 0; 295 int ret; 296 297 /* when the data is dh's source, we should format it */ 298 if ((sg_is_last(data) && len == ctx->key_sz) && 299 ((is_dh && !is_src) || !is_dh)) 300 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); 301 else 302 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); 303 304 if (unlikely(ret)) 305 return ret; 306 307 if (is_src) 308 msg->in = cpu_to_le64(tmp); 309 else 310 msg->out = cpu_to_le64(tmp); 311 312 return 0; 313 } 314 315 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, 316 struct hpre_asym_request *req, 317 struct scatterlist *dst, 318 struct scatterlist *src) 319 { 320 struct device *dev = ctx->dev; 321 struct hpre_sqe *sqe = &req->req; 322 dma_addr_t tmp; 323 324 tmp = le64_to_cpu(sqe->in); 325 if (unlikely(dma_mapping_error(dev, tmp))) 326 return; 327 328 if (src) { 329 if (req->src) 330 dma_free_coherent(dev, ctx->key_sz, req->src, tmp); 331 else 332 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); 333 } 334 335 tmp = le64_to_cpu(sqe->out); 336 if (unlikely(dma_mapping_error(dev, tmp))) 337 return; 338 339 if (req->dst) { 340 if (dst) 341 scatterwalk_map_and_copy(req->dst, dst, 0, 342 ctx->key_sz, 1); 343 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp); 344 } else { 345 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE); 346 } 347 } 348 349 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, 350 void **kreq) 351 { 352 struct hpre_asym_request *req; 353 unsigned int err, done, alg; 354 int id; 355 356 #define HPRE_NO_HW_ERR 0 357 #define HPRE_HW_TASK_DONE 3 358 #define HREE_HW_ERR_MASK GENMASK(10, 0) 359 #define HREE_SQE_DONE_MASK GENMASK(1, 0) 360 #define HREE_ALG_TYPE_MASK GENMASK(4, 0) 361 id = (int)le16_to_cpu(sqe->tag); 362 req = ctx->req_list[id]; 363 hpre_rm_req_from_ctx(req); 364 *kreq = req; 365 366 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & 367 HREE_HW_ERR_MASK; 368 369 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & 370 HREE_SQE_DONE_MASK; 371 372 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) 373 return 0; 374 375 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; 376 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", 377 alg, done, err); 378 379 return -EINVAL; 380 } 381 382 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) 383 { 384 struct hpre *hpre; 385 386 if (!ctx || !qp || qlen < 0) 387 return -EINVAL; 388 389 spin_lock_init(&ctx->req_lock); 390 ctx->qp = qp; 391 ctx->dev = &qp->qm->pdev->dev; 392 393 hpre = container_of(ctx->qp->qm, struct hpre, qm); 394 ctx->hpre = hpre; 395 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); 396 if (!ctx->req_list) 397 return -ENOMEM; 398 ctx->key_sz = 0; 399 ctx->crt_g2_mode = false; 400 idr_init(&ctx->req_idr); 401 402 return 0; 403 } 404 405 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) 406 { 407 if (is_clear_all) { 408 idr_destroy(&ctx->req_idr); 409 kfree(ctx->req_list); 410 hisi_qm_free_qps(&ctx->qp, 1); 411 } 412 413 ctx->crt_g2_mode = false; 414 ctx->key_sz = 0; 415 } 416 417 static bool hpre_is_bd_timeout(struct hpre_asym_request *req, 418 u64 overtime_thrhld) 419 { 420 struct timespec64 reply_time; 421 u64 time_use_us; 422 423 ktime_get_ts64(&reply_time); 424 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) * 425 HPRE_DFX_SEC_TO_US + 426 (reply_time.tv_nsec - req->req_time.tv_nsec) / 427 HPRE_DFX_US_TO_NS; 428 429 if (time_use_us <= overtime_thrhld) 430 return false; 431 432 return true; 433 } 434 435 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp) 436 { 437 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 438 struct hpre_asym_request *req; 439 struct kpp_request *areq; 440 u64 overtime_thrhld; 441 int ret; 442 443 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 444 areq = req->areq.dh; 445 areq->dst_len = ctx->key_sz; 446 447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 450 451 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 452 kpp_request_complete(areq, ret); 453 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 454 } 455 456 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) 457 { 458 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 459 struct hpre_asym_request *req; 460 struct akcipher_request *areq; 461 u64 overtime_thrhld; 462 int ret; 463 464 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 465 466 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 467 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 468 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 469 470 areq = req->areq.rsa; 471 areq->dst_len = ctx->key_sz; 472 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 473 akcipher_request_complete(areq, ret); 474 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 475 } 476 477 static void hpre_alg_cb(struct hisi_qp *qp, void *resp) 478 { 479 struct hpre_ctx *ctx = qp->qp_ctx; 480 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 481 struct hpre_sqe *sqe = resp; 482 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; 483 484 if (unlikely(!req)) { 485 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); 486 return; 487 } 488 489 req->cb(ctx, resp); 490 } 491 492 static void hpre_stop_qp_and_put(struct hisi_qp *qp) 493 { 494 hisi_qm_stop_qp(qp); 495 hisi_qm_free_qps(&qp, 1); 496 } 497 498 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) 499 { 500 struct hisi_qp *qp; 501 int ret; 502 503 qp = hpre_get_qp_and_start(type); 504 if (IS_ERR(qp)) 505 return PTR_ERR(qp); 506 507 qp->qp_ctx = ctx; 508 qp->req_cb = hpre_alg_cb; 509 510 ret = hpre_ctx_set(ctx, qp, qp->sq_depth); 511 if (ret) 512 hpre_stop_qp_and_put(qp); 513 514 return ret; 515 } 516 517 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) 518 { 519 struct hpre_asym_request *h_req; 520 struct hpre_sqe *msg; 521 int req_id; 522 void *tmp; 523 524 if (is_rsa) { 525 struct akcipher_request *akreq = req; 526 527 if (akreq->dst_len < ctx->key_sz) { 528 akreq->dst_len = ctx->key_sz; 529 return -EOVERFLOW; 530 } 531 532 tmp = akcipher_request_ctx(akreq); 533 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 534 h_req->cb = hpre_rsa_cb; 535 h_req->areq.rsa = akreq; 536 msg = &h_req->req; 537 memset(msg, 0, sizeof(*msg)); 538 } else { 539 struct kpp_request *kreq = req; 540 541 if (kreq->dst_len < ctx->key_sz) { 542 kreq->dst_len = ctx->key_sz; 543 return -EOVERFLOW; 544 } 545 546 tmp = kpp_request_ctx(kreq); 547 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 548 h_req->cb = hpre_dh_cb; 549 h_req->areq.dh = kreq; 550 msg = &h_req->req; 551 memset(msg, 0, sizeof(*msg)); 552 msg->key = cpu_to_le64(ctx->dh.dma_xa_p); 553 } 554 555 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 556 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 557 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); 558 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 559 h_req->ctx = ctx; 560 561 req_id = hpre_add_req_to_ctx(h_req); 562 if (req_id < 0) 563 return -EBUSY; 564 565 msg->tag = cpu_to_le16((u16)req_id); 566 567 return 0; 568 } 569 570 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) 571 { 572 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 573 int ctr = 0; 574 int ret; 575 576 do { 577 atomic64_inc(&dfx[HPRE_SEND_CNT].value); 578 spin_lock_bh(&ctx->req_lock); 579 ret = hisi_qp_send(ctx->qp, msg); 580 spin_unlock_bh(&ctx->req_lock); 581 if (ret != -EBUSY) 582 break; 583 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); 584 } while (ctr++ < HPRE_TRY_SEND_TIMES); 585 586 if (likely(!ret)) 587 return ret; 588 589 if (ret != -EBUSY) 590 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value); 591 592 return ret; 593 } 594 595 static int hpre_dh_compute_value(struct kpp_request *req) 596 { 597 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 598 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 599 void *tmp = kpp_request_ctx(req); 600 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 601 struct hpre_sqe *msg = &hpre_req->req; 602 int ret; 603 604 ret = hpre_msg_request_set(ctx, req, false); 605 if (unlikely(ret)) 606 return ret; 607 608 if (req->src) { 609 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); 610 if (unlikely(ret)) 611 goto clear_all; 612 } else { 613 msg->in = cpu_to_le64(ctx->dh.dma_g); 614 } 615 616 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); 617 if (unlikely(ret)) 618 goto clear_all; 619 620 if (ctx->crt_g2_mode && !req->src) 621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2); 622 else 623 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH); 624 625 /* success */ 626 ret = hpre_send(ctx, msg); 627 if (likely(!ret)) 628 return -EINPROGRESS; 629 630 clear_all: 631 hpre_rm_req_from_ctx(hpre_req); 632 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 633 634 return ret; 635 } 636 637 static int hpre_is_dh_params_length_valid(unsigned int key_sz) 638 { 639 #define _HPRE_DH_GRP1 768 640 #define _HPRE_DH_GRP2 1024 641 #define _HPRE_DH_GRP5 1536 642 #define _HPRE_DH_GRP14 2048 643 #define _HPRE_DH_GRP15 3072 644 #define _HPRE_DH_GRP16 4096 645 switch (key_sz) { 646 case _HPRE_DH_GRP1: 647 case _HPRE_DH_GRP2: 648 case _HPRE_DH_GRP5: 649 case _HPRE_DH_GRP14: 650 case _HPRE_DH_GRP15: 651 case _HPRE_DH_GRP16: 652 return 0; 653 default: 654 return -EINVAL; 655 } 656 } 657 658 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) 659 { 660 struct device *dev = ctx->dev; 661 unsigned int sz; 662 663 if (params->p_size > HPRE_DH_MAX_P_SZ) 664 return -EINVAL; 665 666 if (hpre_is_dh_params_length_valid(params->p_size << 667 HPRE_BITS_2_BYTES_SHIFT)) 668 return -EINVAL; 669 670 sz = ctx->key_sz = params->p_size; 671 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, 672 &ctx->dh.dma_xa_p, GFP_KERNEL); 673 if (!ctx->dh.xa_p) 674 return -ENOMEM; 675 676 memcpy(ctx->dh.xa_p + sz, params->p, sz); 677 678 /* If g equals 2 don't copy it */ 679 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) { 680 ctx->crt_g2_mode = true; 681 return 0; 682 } 683 684 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL); 685 if (!ctx->dh.g) { 686 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 687 ctx->dh.dma_xa_p); 688 ctx->dh.xa_p = NULL; 689 return -ENOMEM; 690 } 691 692 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size); 693 694 return 0; 695 } 696 697 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 698 { 699 struct device *dev = ctx->dev; 700 unsigned int sz = ctx->key_sz; 701 702 if (is_clear_all) 703 hisi_qm_stop_qp(ctx->qp); 704 705 if (ctx->dh.g) { 706 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); 707 ctx->dh.g = NULL; 708 } 709 710 if (ctx->dh.xa_p) { 711 memzero_explicit(ctx->dh.xa_p, sz); 712 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 713 ctx->dh.dma_xa_p); 714 ctx->dh.xa_p = NULL; 715 } 716 717 hpre_ctx_clear(ctx, is_clear_all); 718 } 719 720 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, 721 unsigned int len) 722 { 723 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 724 struct dh params; 725 int ret; 726 727 if (crypto_dh_decode_key(buf, len, ¶ms) < 0) 728 return -EINVAL; 729 730 /* Free old secret if any */ 731 hpre_dh_clear_ctx(ctx, false); 732 733 ret = hpre_dh_set_params(ctx, ¶ms); 734 if (ret < 0) 735 goto err_clear_ctx; 736 737 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, 738 params.key_size); 739 740 return 0; 741 742 err_clear_ctx: 743 hpre_dh_clear_ctx(ctx, false); 744 return ret; 745 } 746 747 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) 748 { 749 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 750 751 return ctx->key_sz; 752 } 753 754 static int hpre_dh_init_tfm(struct crypto_kpp *tfm) 755 { 756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 757 758 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 759 760 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 761 } 762 763 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) 764 { 765 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 766 767 hpre_dh_clear_ctx(ctx, true); 768 } 769 770 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) 771 { 772 while (!**ptr && *len) { 773 (*ptr)++; 774 (*len)--; 775 } 776 } 777 778 static bool hpre_rsa_key_size_is_support(unsigned int len) 779 { 780 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT; 781 782 #define _RSA_1024BITS_KEY_WDTH 1024 783 #define _RSA_2048BITS_KEY_WDTH 2048 784 #define _RSA_3072BITS_KEY_WDTH 3072 785 #define _RSA_4096BITS_KEY_WDTH 4096 786 787 switch (bits) { 788 case _RSA_1024BITS_KEY_WDTH: 789 case _RSA_2048BITS_KEY_WDTH: 790 case _RSA_3072BITS_KEY_WDTH: 791 case _RSA_4096BITS_KEY_WDTH: 792 return true; 793 default: 794 return false; 795 } 796 } 797 798 static int hpre_rsa_enc(struct akcipher_request *req) 799 { 800 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 801 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 802 void *tmp = akcipher_request_ctx(req); 803 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 804 struct hpre_sqe *msg = &hpre_req->req; 805 int ret; 806 807 /* For 512 and 1536 bits key size, use soft tfm instead */ 808 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 809 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 810 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 811 ret = crypto_akcipher_encrypt(req); 812 akcipher_request_set_tfm(req, tfm); 813 return ret; 814 } 815 816 if (unlikely(!ctx->rsa.pubkey)) 817 return -EINVAL; 818 819 ret = hpre_msg_request_set(ctx, req, true); 820 if (unlikely(ret)) 821 return ret; 822 823 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); 824 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); 825 826 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 827 if (unlikely(ret)) 828 goto clear_all; 829 830 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 831 if (unlikely(ret)) 832 goto clear_all; 833 834 /* success */ 835 ret = hpre_send(ctx, msg); 836 if (likely(!ret)) 837 return -EINPROGRESS; 838 839 clear_all: 840 hpre_rm_req_from_ctx(hpre_req); 841 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 842 843 return ret; 844 } 845 846 static int hpre_rsa_dec(struct akcipher_request *req) 847 { 848 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 849 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 850 void *tmp = akcipher_request_ctx(req); 851 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 852 struct hpre_sqe *msg = &hpre_req->req; 853 int ret; 854 855 /* For 512 and 1536 bits key size, use soft tfm instead */ 856 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 857 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 858 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 859 ret = crypto_akcipher_decrypt(req); 860 akcipher_request_set_tfm(req, tfm); 861 return ret; 862 } 863 864 if (unlikely(!ctx->rsa.prikey)) 865 return -EINVAL; 866 867 ret = hpre_msg_request_set(ctx, req, true); 868 if (unlikely(ret)) 869 return ret; 870 871 if (ctx->crt_g2_mode) { 872 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); 873 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 874 HPRE_ALG_NC_CRT); 875 } else { 876 msg->key = cpu_to_le64(ctx->rsa.dma_prikey); 877 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 878 HPRE_ALG_NC_NCRT); 879 } 880 881 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 882 if (unlikely(ret)) 883 goto clear_all; 884 885 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 886 if (unlikely(ret)) 887 goto clear_all; 888 889 /* success */ 890 ret = hpre_send(ctx, msg); 891 if (likely(!ret)) 892 return -EINPROGRESS; 893 894 clear_all: 895 hpre_rm_req_from_ctx(hpre_req); 896 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 897 898 return ret; 899 } 900 901 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, 902 size_t vlen, bool private) 903 { 904 const char *ptr = value; 905 906 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 907 908 ctx->key_sz = vlen; 909 910 /* if invalid key size provided, we use software tfm */ 911 if (!hpre_rsa_key_size_is_support(ctx->key_sz)) 912 return 0; 913 914 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, 915 &ctx->rsa.dma_pubkey, 916 GFP_KERNEL); 917 if (!ctx->rsa.pubkey) 918 return -ENOMEM; 919 920 if (private) { 921 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, 922 &ctx->rsa.dma_prikey, 923 GFP_KERNEL); 924 if (!ctx->rsa.prikey) { 925 dma_free_coherent(ctx->dev, vlen << 1, 926 ctx->rsa.pubkey, 927 ctx->rsa.dma_pubkey); 928 ctx->rsa.pubkey = NULL; 929 return -ENOMEM; 930 } 931 memcpy(ctx->rsa.prikey + vlen, ptr, vlen); 932 } 933 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen); 934 935 /* Using hardware HPRE to do RSA */ 936 return 1; 937 } 938 939 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value, 940 size_t vlen) 941 { 942 const char *ptr = value; 943 944 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 945 946 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 947 return -EINVAL; 948 949 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen); 950 951 return 0; 952 } 953 954 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value, 955 size_t vlen) 956 { 957 const char *ptr = value; 958 959 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 960 961 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 962 return -EINVAL; 963 964 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen); 965 966 return 0; 967 } 968 969 static int hpre_crt_para_get(char *para, size_t para_sz, 970 const char *raw, size_t raw_sz) 971 { 972 const char *ptr = raw; 973 size_t len = raw_sz; 974 975 hpre_rsa_drop_leading_zeros(&ptr, &len); 976 if (!len || len > para_sz) 977 return -EINVAL; 978 979 memcpy(para + para_sz - len, ptr, len); 980 981 return 0; 982 } 983 984 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) 985 { 986 unsigned int hlf_ksz = ctx->key_sz >> 1; 987 struct device *dev = ctx->dev; 988 u64 offset; 989 int ret; 990 991 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, 992 &ctx->rsa.dma_crt_prikey, 993 GFP_KERNEL); 994 if (!ctx->rsa.crt_prikey) 995 return -ENOMEM; 996 997 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz, 998 rsa_key->dq, rsa_key->dq_sz); 999 if (ret) 1000 goto free_key; 1001 1002 offset = hlf_ksz; 1003 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1004 rsa_key->dp, rsa_key->dp_sz); 1005 if (ret) 1006 goto free_key; 1007 1008 offset = hlf_ksz * HPRE_CRT_Q; 1009 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1010 rsa_key->q, rsa_key->q_sz); 1011 if (ret) 1012 goto free_key; 1013 1014 offset = hlf_ksz * HPRE_CRT_P; 1015 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1016 rsa_key->p, rsa_key->p_sz); 1017 if (ret) 1018 goto free_key; 1019 1020 offset = hlf_ksz * HPRE_CRT_INV; 1021 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1022 rsa_key->qinv, rsa_key->qinv_sz); 1023 if (ret) 1024 goto free_key; 1025 1026 ctx->crt_g2_mode = true; 1027 1028 return 0; 1029 1030 free_key: 1031 offset = hlf_ksz * HPRE_CRT_PRMS; 1032 memzero_explicit(ctx->rsa.crt_prikey, offset); 1033 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey, 1034 ctx->rsa.dma_crt_prikey); 1035 ctx->rsa.crt_prikey = NULL; 1036 ctx->crt_g2_mode = false; 1037 1038 return ret; 1039 } 1040 1041 /* If it is clear all, all the resources of the QP will be cleaned. */ 1042 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 1043 { 1044 unsigned int half_key_sz = ctx->key_sz >> 1; 1045 struct device *dev = ctx->dev; 1046 1047 if (is_clear_all) 1048 hisi_qm_stop_qp(ctx->qp); 1049 1050 if (ctx->rsa.pubkey) { 1051 dma_free_coherent(dev, ctx->key_sz << 1, 1052 ctx->rsa.pubkey, ctx->rsa.dma_pubkey); 1053 ctx->rsa.pubkey = NULL; 1054 } 1055 1056 if (ctx->rsa.crt_prikey) { 1057 memzero_explicit(ctx->rsa.crt_prikey, 1058 half_key_sz * HPRE_CRT_PRMS); 1059 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS, 1060 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey); 1061 ctx->rsa.crt_prikey = NULL; 1062 } 1063 1064 if (ctx->rsa.prikey) { 1065 memzero_explicit(ctx->rsa.prikey, ctx->key_sz); 1066 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey, 1067 ctx->rsa.dma_prikey); 1068 ctx->rsa.prikey = NULL; 1069 } 1070 1071 hpre_ctx_clear(ctx, is_clear_all); 1072 } 1073 1074 /* 1075 * we should judge if it is CRT or not, 1076 * CRT: return true, N-CRT: return false . 1077 */ 1078 static bool hpre_is_crt_key(struct rsa_key *key) 1079 { 1080 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz + 1081 key->qinv_sz; 1082 1083 #define LEN_OF_NCRT_PARA 5 1084 1085 /* N-CRT less than 5 parameters */ 1086 return len > LEN_OF_NCRT_PARA; 1087 } 1088 1089 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, 1090 unsigned int keylen, bool private) 1091 { 1092 struct rsa_key rsa_key; 1093 int ret; 1094 1095 hpre_rsa_clear_ctx(ctx, false); 1096 1097 if (private) 1098 ret = rsa_parse_priv_key(&rsa_key, key, keylen); 1099 else 1100 ret = rsa_parse_pub_key(&rsa_key, key, keylen); 1101 if (ret < 0) 1102 return ret; 1103 1104 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private); 1105 if (ret <= 0) 1106 return ret; 1107 1108 if (private) { 1109 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); 1110 if (ret < 0) 1111 goto free; 1112 1113 if (hpre_is_crt_key(&rsa_key)) { 1114 ret = hpre_rsa_setkey_crt(ctx, &rsa_key); 1115 if (ret < 0) 1116 goto free; 1117 } 1118 } 1119 1120 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); 1121 if (ret < 0) 1122 goto free; 1123 1124 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) { 1125 ret = -EINVAL; 1126 goto free; 1127 } 1128 1129 return 0; 1130 1131 free: 1132 hpre_rsa_clear_ctx(ctx, false); 1133 return ret; 1134 } 1135 1136 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, 1137 unsigned int keylen) 1138 { 1139 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1140 int ret; 1141 1142 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen); 1143 if (ret) 1144 return ret; 1145 1146 return hpre_rsa_setkey(ctx, key, keylen, false); 1147 } 1148 1149 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, 1150 unsigned int keylen) 1151 { 1152 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1153 int ret; 1154 1155 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen); 1156 if (ret) 1157 return ret; 1158 1159 return hpre_rsa_setkey(ctx, key, keylen, true); 1160 } 1161 1162 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) 1163 { 1164 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1165 1166 /* For 512 and 1536 bits key size, use soft tfm instead */ 1167 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 1168 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) 1169 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); 1170 1171 return ctx->key_sz; 1172 } 1173 1174 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) 1175 { 1176 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1177 int ret; 1178 1179 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0); 1180 if (IS_ERR(ctx->rsa.soft_tfm)) { 1181 pr_err("Can not alloc_akcipher!\n"); 1182 return PTR_ERR(ctx->rsa.soft_tfm); 1183 } 1184 1185 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) + 1186 hpre_align_pd()); 1187 1188 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 1189 if (ret) 1190 crypto_free_akcipher(ctx->rsa.soft_tfm); 1191 1192 return ret; 1193 } 1194 1195 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) 1196 { 1197 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1198 1199 hpre_rsa_clear_ctx(ctx, true); 1200 crypto_free_akcipher(ctx->rsa.soft_tfm); 1201 } 1202 1203 static void hpre_key_to_big_end(u8 *data, int len) 1204 { 1205 int i, j; 1206 1207 for (i = 0; i < len / 2; i++) { 1208 j = len - i - 1; 1209 swap(data[j], data[i]); 1210 } 1211 } 1212 1213 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, 1214 bool is_ecdh) 1215 { 1216 struct device *dev = ctx->dev; 1217 unsigned int sz = ctx->key_sz; 1218 unsigned int shift = sz << 1; 1219 1220 if (is_clear_all) 1221 hisi_qm_stop_qp(ctx->qp); 1222 1223 if (is_ecdh && ctx->ecdh.p) { 1224 /* ecdh: p->a->k->b */ 1225 memzero_explicit(ctx->ecdh.p + shift, sz); 1226 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1227 ctx->ecdh.p = NULL; 1228 } else if (!is_ecdh && ctx->curve25519.p) { 1229 /* curve25519: p->a->k */ 1230 memzero_explicit(ctx->curve25519.p + shift, sz); 1231 dma_free_coherent(dev, sz << 2, ctx->curve25519.p, 1232 ctx->curve25519.dma_p); 1233 ctx->curve25519.p = NULL; 1234 } 1235 1236 hpre_ctx_clear(ctx, is_clear_all); 1237 } 1238 1239 /* 1240 * The bits of 192/224/256/384/521 are supported by HPRE, 1241 * and convert the bits like: 1242 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576; 1243 * If the parameter bit width is insufficient, then we fill in the 1244 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8; 1245 */ 1246 static unsigned int hpre_ecdh_supported_curve(unsigned short id) 1247 { 1248 switch (id) { 1249 case ECC_CURVE_NIST_P192: 1250 case ECC_CURVE_NIST_P256: 1251 return HPRE_ECC_HW256_KSZ_B; 1252 case ECC_CURVE_NIST_P384: 1253 return HPRE_ECC_HW384_KSZ_B; 1254 default: 1255 break; 1256 } 1257 1258 return 0; 1259 } 1260 1261 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits) 1262 { 1263 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64); 1264 u8 i = 0; 1265 1266 while (i < ndigits - 1) { 1267 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64)); 1268 i++; 1269 } 1270 1271 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz); 1272 hpre_key_to_big_end((u8 *)addr, cur_sz); 1273 } 1274 1275 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params, 1276 unsigned int cur_sz) 1277 { 1278 unsigned int shifta = ctx->key_sz << 1; 1279 unsigned int shiftb = ctx->key_sz << 2; 1280 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz; 1281 void *a = ctx->ecdh.p + shifta - cur_sz; 1282 void *b = ctx->ecdh.p + shiftb - cur_sz; 1283 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz; 1284 void *y = ctx->ecdh.g + shifta - cur_sz; 1285 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id); 1286 char *n; 1287 1288 if (unlikely(!curve)) 1289 return -EINVAL; 1290 1291 n = kzalloc(ctx->key_sz, GFP_KERNEL); 1292 if (!n) 1293 return -ENOMEM; 1294 1295 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits); 1296 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits); 1297 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits); 1298 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits); 1299 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits); 1300 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits); 1301 1302 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) { 1303 kfree(n); 1304 return -EINVAL; 1305 } 1306 1307 kfree(n); 1308 return 0; 1309 } 1310 1311 static unsigned int hpre_ecdh_get_curvesz(unsigned short id) 1312 { 1313 switch (id) { 1314 case ECC_CURVE_NIST_P192: 1315 return HPRE_ECC_NIST_P192_N_SIZE; 1316 case ECC_CURVE_NIST_P256: 1317 return HPRE_ECC_NIST_P256_N_SIZE; 1318 case ECC_CURVE_NIST_P384: 1319 return HPRE_ECC_NIST_P384_N_SIZE; 1320 default: 1321 break; 1322 } 1323 1324 return 0; 1325 } 1326 1327 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) 1328 { 1329 struct device *dev = ctx->dev; 1330 unsigned int sz, shift, curve_sz; 1331 int ret; 1332 1333 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id); 1334 if (!ctx->key_sz) 1335 return -EINVAL; 1336 1337 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1338 if (!curve_sz || params->key_size > curve_sz) 1339 return -EINVAL; 1340 1341 sz = ctx->key_sz; 1342 1343 if (!ctx->ecdh.p) { 1344 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p, 1345 GFP_KERNEL); 1346 if (!ctx->ecdh.p) 1347 return -ENOMEM; 1348 } 1349 1350 shift = sz << 2; 1351 ctx->ecdh.g = ctx->ecdh.p + shift; 1352 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift; 1353 1354 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz); 1355 if (ret) { 1356 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret); 1357 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1358 ctx->ecdh.p = NULL; 1359 return ret; 1360 } 1361 1362 return 0; 1363 } 1364 1365 static bool hpre_key_is_zero(char *key, unsigned short key_sz) 1366 { 1367 int i; 1368 1369 for (i = 0; i < key_sz; i++) 1370 if (key[i]) 1371 return false; 1372 1373 return true; 1374 } 1375 1376 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) 1377 { 1378 struct device *dev = ctx->dev; 1379 int ret; 1380 1381 ret = crypto_get_default_rng(); 1382 if (ret) { 1383 dev_err(dev, "failed to get default rng, ret = %d!\n", ret); 1384 return ret; 1385 } 1386 1387 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key, 1388 params->key_size); 1389 crypto_put_default_rng(); 1390 if (ret) 1391 dev_err(dev, "failed to get rng, ret = %d!\n", ret); 1392 1393 return ret; 1394 } 1395 1396 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, 1397 unsigned int len) 1398 { 1399 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1400 unsigned int sz, sz_shift, curve_sz; 1401 struct device *dev = ctx->dev; 1402 char key[HPRE_ECC_MAX_KSZ]; 1403 struct ecdh params; 1404 int ret; 1405 1406 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { 1407 dev_err(dev, "failed to decode ecdh key!\n"); 1408 return -EINVAL; 1409 } 1410 1411 /* Use stdrng to generate private key */ 1412 if (!params.key || !params.key_size) { 1413 params.key = key; 1414 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1415 if (!curve_sz) { 1416 dev_err(dev, "Invalid curve size!\n"); 1417 return -EINVAL; 1418 } 1419 1420 params.key_size = curve_sz - 1; 1421 ret = ecdh_gen_privkey(ctx, ¶ms); 1422 if (ret) 1423 return ret; 1424 } 1425 1426 if (hpre_key_is_zero(params.key, params.key_size)) { 1427 dev_err(dev, "Invalid hpre key!\n"); 1428 return -EINVAL; 1429 } 1430 1431 hpre_ecc_clear_ctx(ctx, false, true); 1432 1433 ret = hpre_ecdh_set_param(ctx, ¶ms); 1434 if (ret < 0) { 1435 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret); 1436 return ret; 1437 } 1438 1439 sz = ctx->key_sz; 1440 sz_shift = (sz << 1) + sz - params.key_size; 1441 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size); 1442 1443 return 0; 1444 } 1445 1446 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, 1447 struct hpre_asym_request *req, 1448 struct scatterlist *dst, 1449 struct scatterlist *src) 1450 { 1451 struct device *dev = ctx->dev; 1452 struct hpre_sqe *sqe = &req->req; 1453 dma_addr_t dma; 1454 1455 dma = le64_to_cpu(sqe->in); 1456 if (unlikely(dma_mapping_error(dev, dma))) 1457 return; 1458 1459 if (src && req->src) 1460 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); 1461 1462 dma = le64_to_cpu(sqe->out); 1463 if (unlikely(dma_mapping_error(dev, dma))) 1464 return; 1465 1466 if (req->dst) 1467 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); 1468 if (dst) 1469 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE); 1470 } 1471 1472 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) 1473 { 1474 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1475 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1476 struct hpre_asym_request *req = NULL; 1477 struct kpp_request *areq; 1478 u64 overtime_thrhld; 1479 char *p; 1480 int ret; 1481 1482 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1483 areq = req->areq.ecdh; 1484 areq->dst_len = ctx->key_sz << 1; 1485 1486 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1487 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1488 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1489 1490 p = sg_virt(areq->dst); 1491 memmove(p, p + ctx->key_sz - curve_sz, curve_sz); 1492 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); 1493 1494 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1495 kpp_request_complete(areq, ret); 1496 1497 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1498 } 1499 1500 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, 1501 struct kpp_request *req) 1502 { 1503 struct hpre_asym_request *h_req; 1504 struct hpre_sqe *msg; 1505 int req_id; 1506 void *tmp; 1507 1508 if (req->dst_len < ctx->key_sz << 1) { 1509 req->dst_len = ctx->key_sz << 1; 1510 return -EINVAL; 1511 } 1512 1513 tmp = kpp_request_ctx(req); 1514 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 1515 h_req->cb = hpre_ecdh_cb; 1516 h_req->areq.ecdh = req; 1517 msg = &h_req->req; 1518 memset(msg, 0, sizeof(*msg)); 1519 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1520 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1521 msg->key = cpu_to_le64(ctx->ecdh.dma_p); 1522 1523 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1524 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1525 h_req->ctx = ctx; 1526 1527 req_id = hpre_add_req_to_ctx(h_req); 1528 if (req_id < 0) 1529 return -EBUSY; 1530 1531 msg->tag = cpu_to_le16((u16)req_id); 1532 return 0; 1533 } 1534 1535 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, 1536 struct scatterlist *data, unsigned int len) 1537 { 1538 struct hpre_sqe *msg = &hpre_req->req; 1539 struct hpre_ctx *ctx = hpre_req->ctx; 1540 struct device *dev = ctx->dev; 1541 unsigned int tmpshift; 1542 dma_addr_t dma = 0; 1543 void *ptr; 1544 int shift; 1545 1546 /* Src_data include gx and gy. */ 1547 shift = ctx->key_sz - (len >> 1); 1548 if (unlikely(shift < 0)) 1549 return -EINVAL; 1550 1551 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL); 1552 if (unlikely(!ptr)) 1553 return -ENOMEM; 1554 1555 tmpshift = ctx->key_sz << 1; 1556 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0); 1557 memcpy(ptr + shift, ptr + tmpshift, len >> 1); 1558 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1); 1559 1560 hpre_req->src = ptr; 1561 msg->in = cpu_to_le64(dma); 1562 return 0; 1563 } 1564 1565 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, 1566 struct scatterlist *data, unsigned int len) 1567 { 1568 struct hpre_sqe *msg = &hpre_req->req; 1569 struct hpre_ctx *ctx = hpre_req->ctx; 1570 struct device *dev = ctx->dev; 1571 dma_addr_t dma; 1572 1573 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { 1574 dev_err(dev, "data or data length is illegal!\n"); 1575 return -EINVAL; 1576 } 1577 1578 hpre_req->dst = NULL; 1579 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1580 if (unlikely(dma_mapping_error(dev, dma))) { 1581 dev_err(dev, "dma map data err!\n"); 1582 return -ENOMEM; 1583 } 1584 1585 msg->out = cpu_to_le64(dma); 1586 return 0; 1587 } 1588 1589 static int hpre_ecdh_compute_value(struct kpp_request *req) 1590 { 1591 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1592 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1593 struct device *dev = ctx->dev; 1594 void *tmp = kpp_request_ctx(req); 1595 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 1596 struct hpre_sqe *msg = &hpre_req->req; 1597 int ret; 1598 1599 ret = hpre_ecdh_msg_request_set(ctx, req); 1600 if (unlikely(ret)) { 1601 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret); 1602 return ret; 1603 } 1604 1605 if (req->src) { 1606 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len); 1607 if (unlikely(ret)) { 1608 dev_err(dev, "failed to init src data, ret = %d!\n", ret); 1609 goto clear_all; 1610 } 1611 } else { 1612 msg->in = cpu_to_le64(ctx->ecdh.dma_g); 1613 } 1614 1615 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len); 1616 if (unlikely(ret)) { 1617 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1618 goto clear_all; 1619 } 1620 1621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); 1622 ret = hpre_send(ctx, msg); 1623 if (likely(!ret)) 1624 return -EINPROGRESS; 1625 1626 clear_all: 1627 hpre_rm_req_from_ctx(hpre_req); 1628 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1629 return ret; 1630 } 1631 1632 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) 1633 { 1634 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1635 1636 /* max size is the pub_key_size, include x and y */ 1637 return ctx->key_sz << 1; 1638 } 1639 1640 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) 1641 { 1642 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1643 1644 ctx->curve_id = ECC_CURVE_NIST_P192; 1645 1646 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1647 1648 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1649 } 1650 1651 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) 1652 { 1653 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1654 1655 ctx->curve_id = ECC_CURVE_NIST_P256; 1656 1657 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1658 1659 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1660 } 1661 1662 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) 1663 { 1664 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1665 1666 ctx->curve_id = ECC_CURVE_NIST_P384; 1667 1668 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1669 1670 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1671 } 1672 1673 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) 1674 { 1675 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1676 1677 hpre_ecc_clear_ctx(ctx, true, true); 1678 } 1679 1680 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, 1681 unsigned int len) 1682 { 1683 u8 secret[CURVE25519_KEY_SIZE] = { 0 }; 1684 unsigned int sz = ctx->key_sz; 1685 const struct ecc_curve *curve; 1686 unsigned int shift = sz << 1; 1687 void *p; 1688 1689 /* 1690 * The key from 'buf' is in little-endian, we should preprocess it as 1691 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64", 1692 * then convert it to big endian. Only in this way, the result can be 1693 * the same as the software curve-25519 that exists in crypto. 1694 */ 1695 memcpy(secret, buf, len); 1696 curve25519_clamp_secret(secret); 1697 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE); 1698 1699 p = ctx->curve25519.p + sz - len; 1700 1701 curve = ecc_get_curve25519(); 1702 1703 /* fill curve parameters */ 1704 fill_curve_param(p, curve->p, len, curve->g.ndigits); 1705 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits); 1706 memcpy(p + shift, secret, len); 1707 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits); 1708 memzero_explicit(secret, CURVE25519_KEY_SIZE); 1709 } 1710 1711 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, 1712 unsigned int len) 1713 { 1714 struct device *dev = ctx->dev; 1715 unsigned int sz = ctx->key_sz; 1716 unsigned int shift = sz << 1; 1717 1718 /* p->a->k->gx */ 1719 if (!ctx->curve25519.p) { 1720 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2, 1721 &ctx->curve25519.dma_p, 1722 GFP_KERNEL); 1723 if (!ctx->curve25519.p) 1724 return -ENOMEM; 1725 } 1726 1727 ctx->curve25519.g = ctx->curve25519.p + shift + sz; 1728 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz; 1729 1730 hpre_curve25519_fill_curve(ctx, buf, len); 1731 1732 return 0; 1733 } 1734 1735 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, 1736 unsigned int len) 1737 { 1738 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1739 struct device *dev = ctx->dev; 1740 int ret = -EINVAL; 1741 1742 if (len != CURVE25519_KEY_SIZE || 1743 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) { 1744 dev_err(dev, "key is null or key len is not 32bytes!\n"); 1745 return ret; 1746 } 1747 1748 /* Free old secret if any */ 1749 hpre_ecc_clear_ctx(ctx, false, false); 1750 1751 ctx->key_sz = CURVE25519_KEY_SIZE; 1752 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE); 1753 if (ret) { 1754 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret); 1755 hpre_ecc_clear_ctx(ctx, false, false); 1756 return ret; 1757 } 1758 1759 return 0; 1760 } 1761 1762 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, 1763 struct hpre_asym_request *req, 1764 struct scatterlist *dst, 1765 struct scatterlist *src) 1766 { 1767 struct device *dev = ctx->dev; 1768 struct hpre_sqe *sqe = &req->req; 1769 dma_addr_t dma; 1770 1771 dma = le64_to_cpu(sqe->in); 1772 if (unlikely(dma_mapping_error(dev, dma))) 1773 return; 1774 1775 if (src && req->src) 1776 dma_free_coherent(dev, ctx->key_sz, req->src, dma); 1777 1778 dma = le64_to_cpu(sqe->out); 1779 if (unlikely(dma_mapping_error(dev, dma))) 1780 return; 1781 1782 if (req->dst) 1783 dma_free_coherent(dev, ctx->key_sz, req->dst, dma); 1784 if (dst) 1785 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE); 1786 } 1787 1788 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) 1789 { 1790 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1791 struct hpre_asym_request *req = NULL; 1792 struct kpp_request *areq; 1793 u64 overtime_thrhld; 1794 int ret; 1795 1796 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1797 areq = req->areq.curve25519; 1798 areq->dst_len = ctx->key_sz; 1799 1800 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1801 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1802 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1803 1804 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); 1805 1806 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1807 kpp_request_complete(areq, ret); 1808 1809 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1810 } 1811 1812 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, 1813 struct kpp_request *req) 1814 { 1815 struct hpre_asym_request *h_req; 1816 struct hpre_sqe *msg; 1817 int req_id; 1818 void *tmp; 1819 1820 if (unlikely(req->dst_len < ctx->key_sz)) { 1821 req->dst_len = ctx->key_sz; 1822 return -EINVAL; 1823 } 1824 1825 tmp = kpp_request_ctx(req); 1826 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 1827 h_req->cb = hpre_curve25519_cb; 1828 h_req->areq.curve25519 = req; 1829 msg = &h_req->req; 1830 memset(msg, 0, sizeof(*msg)); 1831 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1832 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1833 msg->key = cpu_to_le64(ctx->curve25519.dma_p); 1834 1835 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1836 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1837 h_req->ctx = ctx; 1838 1839 req_id = hpre_add_req_to_ctx(h_req); 1840 if (req_id < 0) 1841 return -EBUSY; 1842 1843 msg->tag = cpu_to_le16((u16)req_id); 1844 return 0; 1845 } 1846 1847 static void hpre_curve25519_src_modulo_p(u8 *ptr) 1848 { 1849 int i; 1850 1851 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++) 1852 ptr[i] = 0; 1853 1854 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */ 1855 ptr[i] -= 0xed; 1856 } 1857 1858 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, 1859 struct scatterlist *data, unsigned int len) 1860 { 1861 struct hpre_sqe *msg = &hpre_req->req; 1862 struct hpre_ctx *ctx = hpre_req->ctx; 1863 struct device *dev = ctx->dev; 1864 u8 p[CURVE25519_KEY_SIZE] = { 0 }; 1865 const struct ecc_curve *curve; 1866 dma_addr_t dma = 0; 1867 u8 *ptr; 1868 1869 if (len != CURVE25519_KEY_SIZE) { 1870 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len); 1871 return -EINVAL; 1872 } 1873 1874 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL); 1875 if (unlikely(!ptr)) 1876 return -ENOMEM; 1877 1878 scatterwalk_map_and_copy(ptr, data, 0, len, 0); 1879 1880 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) { 1881 dev_err(dev, "gx is null!\n"); 1882 goto err; 1883 } 1884 1885 /* 1886 * Src_data(gx) is in little-endian order, MSB in the final byte should 1887 * be masked as described in RFC7748, then transform it to big-endian 1888 * form, then hisi_hpre can use the data. 1889 */ 1890 ptr[31] &= 0x7f; 1891 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE); 1892 1893 curve = ecc_get_curve25519(); 1894 1895 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits); 1896 1897 /* 1898 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, 1899 * we get its modulus to p, and then use it. 1900 */ 1901 if (memcmp(ptr, p, ctx->key_sz) == 0) { 1902 dev_err(dev, "gx is p!\n"); 1903 goto err; 1904 } else if (memcmp(ptr, p, ctx->key_sz) > 0) { 1905 hpre_curve25519_src_modulo_p(ptr); 1906 } 1907 1908 hpre_req->src = ptr; 1909 msg->in = cpu_to_le64(dma); 1910 return 0; 1911 1912 err: 1913 dma_free_coherent(dev, ctx->key_sz, ptr, dma); 1914 return -EINVAL; 1915 } 1916 1917 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, 1918 struct scatterlist *data, unsigned int len) 1919 { 1920 struct hpre_sqe *msg = &hpre_req->req; 1921 struct hpre_ctx *ctx = hpre_req->ctx; 1922 struct device *dev = ctx->dev; 1923 dma_addr_t dma; 1924 1925 if (!data || !sg_is_last(data) || len != ctx->key_sz) { 1926 dev_err(dev, "data or data length is illegal!\n"); 1927 return -EINVAL; 1928 } 1929 1930 hpre_req->dst = NULL; 1931 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1932 if (unlikely(dma_mapping_error(dev, dma))) { 1933 dev_err(dev, "dma map data err!\n"); 1934 return -ENOMEM; 1935 } 1936 1937 msg->out = cpu_to_le64(dma); 1938 return 0; 1939 } 1940 1941 static int hpre_curve25519_compute_value(struct kpp_request *req) 1942 { 1943 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1944 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1945 struct device *dev = ctx->dev; 1946 void *tmp = kpp_request_ctx(req); 1947 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 1948 struct hpre_sqe *msg = &hpre_req->req; 1949 int ret; 1950 1951 ret = hpre_curve25519_msg_request_set(ctx, req); 1952 if (unlikely(ret)) { 1953 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret); 1954 return ret; 1955 } 1956 1957 if (req->src) { 1958 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len); 1959 if (unlikely(ret)) { 1960 dev_err(dev, "failed to init src data, ret = %d!\n", 1961 ret); 1962 goto clear_all; 1963 } 1964 } else { 1965 msg->in = cpu_to_le64(ctx->curve25519.dma_g); 1966 } 1967 1968 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len); 1969 if (unlikely(ret)) { 1970 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1971 goto clear_all; 1972 } 1973 1974 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL); 1975 ret = hpre_send(ctx, msg); 1976 if (likely(!ret)) 1977 return -EINPROGRESS; 1978 1979 clear_all: 1980 hpre_rm_req_from_ctx(hpre_req); 1981 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1982 return ret; 1983 } 1984 1985 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm) 1986 { 1987 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1988 1989 return ctx->key_sz; 1990 } 1991 1992 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm) 1993 { 1994 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1995 1996 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1997 1998 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1999 } 2000 2001 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm) 2002 { 2003 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 2004 2005 hpre_ecc_clear_ctx(ctx, true, false); 2006 } 2007 2008 static struct akcipher_alg rsa = { 2009 .encrypt = hpre_rsa_enc, 2010 .decrypt = hpre_rsa_dec, 2011 .set_pub_key = hpre_rsa_setpubkey, 2012 .set_priv_key = hpre_rsa_setprivkey, 2013 .max_size = hpre_rsa_max_size, 2014 .init = hpre_rsa_init_tfm, 2015 .exit = hpre_rsa_exit_tfm, 2016 .base = { 2017 .cra_ctxsize = sizeof(struct hpre_ctx), 2018 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2019 .cra_name = "rsa", 2020 .cra_driver_name = "hpre-rsa", 2021 .cra_module = THIS_MODULE, 2022 }, 2023 }; 2024 2025 static struct kpp_alg dh = { 2026 .set_secret = hpre_dh_set_secret, 2027 .generate_public_key = hpre_dh_compute_value, 2028 .compute_shared_secret = hpre_dh_compute_value, 2029 .max_size = hpre_dh_max_size, 2030 .init = hpre_dh_init_tfm, 2031 .exit = hpre_dh_exit_tfm, 2032 .base = { 2033 .cra_ctxsize = sizeof(struct hpre_ctx), 2034 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2035 .cra_name = "dh", 2036 .cra_driver_name = "hpre-dh", 2037 .cra_module = THIS_MODULE, 2038 }, 2039 }; 2040 2041 static struct kpp_alg ecdh_curves[] = { 2042 { 2043 .set_secret = hpre_ecdh_set_secret, 2044 .generate_public_key = hpre_ecdh_compute_value, 2045 .compute_shared_secret = hpre_ecdh_compute_value, 2046 .max_size = hpre_ecdh_max_size, 2047 .init = hpre_ecdh_nist_p192_init_tfm, 2048 .exit = hpre_ecdh_exit_tfm, 2049 .base = { 2050 .cra_ctxsize = sizeof(struct hpre_ctx), 2051 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2052 .cra_name = "ecdh-nist-p192", 2053 .cra_driver_name = "hpre-ecdh-nist-p192", 2054 .cra_module = THIS_MODULE, 2055 }, 2056 }, { 2057 .set_secret = hpre_ecdh_set_secret, 2058 .generate_public_key = hpre_ecdh_compute_value, 2059 .compute_shared_secret = hpre_ecdh_compute_value, 2060 .max_size = hpre_ecdh_max_size, 2061 .init = hpre_ecdh_nist_p256_init_tfm, 2062 .exit = hpre_ecdh_exit_tfm, 2063 .base = { 2064 .cra_ctxsize = sizeof(struct hpre_ctx), 2065 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2066 .cra_name = "ecdh-nist-p256", 2067 .cra_driver_name = "hpre-ecdh-nist-p256", 2068 .cra_module = THIS_MODULE, 2069 }, 2070 }, { 2071 .set_secret = hpre_ecdh_set_secret, 2072 .generate_public_key = hpre_ecdh_compute_value, 2073 .compute_shared_secret = hpre_ecdh_compute_value, 2074 .max_size = hpre_ecdh_max_size, 2075 .init = hpre_ecdh_nist_p384_init_tfm, 2076 .exit = hpre_ecdh_exit_tfm, 2077 .base = { 2078 .cra_ctxsize = sizeof(struct hpre_ctx), 2079 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2080 .cra_name = "ecdh-nist-p384", 2081 .cra_driver_name = "hpre-ecdh-nist-p384", 2082 .cra_module = THIS_MODULE, 2083 }, 2084 } 2085 }; 2086 2087 static struct kpp_alg curve25519_alg = { 2088 .set_secret = hpre_curve25519_set_secret, 2089 .generate_public_key = hpre_curve25519_compute_value, 2090 .compute_shared_secret = hpre_curve25519_compute_value, 2091 .max_size = hpre_curve25519_max_size, 2092 .init = hpre_curve25519_init_tfm, 2093 .exit = hpre_curve25519_exit_tfm, 2094 .base = { 2095 .cra_ctxsize = sizeof(struct hpre_ctx), 2096 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2097 .cra_name = "curve25519", 2098 .cra_driver_name = "hpre-curve25519", 2099 .cra_module = THIS_MODULE, 2100 }, 2101 }; 2102 2103 static int hpre_register_rsa(struct hisi_qm *qm) 2104 { 2105 int ret; 2106 2107 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) 2108 return 0; 2109 2110 rsa.base.cra_flags = 0; 2111 ret = crypto_register_akcipher(&rsa); 2112 if (ret) 2113 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret); 2114 2115 return ret; 2116 } 2117 2118 static void hpre_unregister_rsa(struct hisi_qm *qm) 2119 { 2120 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) 2121 return; 2122 2123 crypto_unregister_akcipher(&rsa); 2124 } 2125 2126 static int hpre_register_dh(struct hisi_qm *qm) 2127 { 2128 int ret; 2129 2130 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) 2131 return 0; 2132 2133 ret = crypto_register_kpp(&dh); 2134 if (ret) 2135 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret); 2136 2137 return ret; 2138 } 2139 2140 static void hpre_unregister_dh(struct hisi_qm *qm) 2141 { 2142 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) 2143 return; 2144 2145 crypto_unregister_kpp(&dh); 2146 } 2147 2148 static int hpre_register_ecdh(struct hisi_qm *qm) 2149 { 2150 int ret, i; 2151 2152 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) 2153 return 0; 2154 2155 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) { 2156 ret = crypto_register_kpp(&ecdh_curves[i]); 2157 if (ret) { 2158 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n", 2159 ecdh_curves[i].base.cra_name, ret); 2160 goto unreg_kpp; 2161 } 2162 } 2163 2164 return 0; 2165 2166 unreg_kpp: 2167 for (--i; i >= 0; --i) 2168 crypto_unregister_kpp(&ecdh_curves[i]); 2169 2170 return ret; 2171 } 2172 2173 static void hpre_unregister_ecdh(struct hisi_qm *qm) 2174 { 2175 int i; 2176 2177 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) 2178 return; 2179 2180 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i) 2181 crypto_unregister_kpp(&ecdh_curves[i]); 2182 } 2183 2184 static int hpre_register_x25519(struct hisi_qm *qm) 2185 { 2186 int ret; 2187 2188 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) 2189 return 0; 2190 2191 ret = crypto_register_kpp(&curve25519_alg); 2192 if (ret) 2193 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret); 2194 2195 return ret; 2196 } 2197 2198 static void hpre_unregister_x25519(struct hisi_qm *qm) 2199 { 2200 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) 2201 return; 2202 2203 crypto_unregister_kpp(&curve25519_alg); 2204 } 2205 2206 int hpre_algs_register(struct hisi_qm *qm) 2207 { 2208 int ret = 0; 2209 2210 mutex_lock(&hpre_algs_lock); 2211 if (hpre_available_devs) { 2212 hpre_available_devs++; 2213 goto unlock; 2214 } 2215 2216 ret = hpre_register_rsa(qm); 2217 if (ret) 2218 goto unlock; 2219 2220 ret = hpre_register_dh(qm); 2221 if (ret) 2222 goto unreg_rsa; 2223 2224 ret = hpre_register_ecdh(qm); 2225 if (ret) 2226 goto unreg_dh; 2227 2228 ret = hpre_register_x25519(qm); 2229 if (ret) 2230 goto unreg_ecdh; 2231 2232 hpre_available_devs++; 2233 mutex_unlock(&hpre_algs_lock); 2234 2235 return ret; 2236 2237 unreg_ecdh: 2238 hpre_unregister_ecdh(qm); 2239 unreg_dh: 2240 hpre_unregister_dh(qm); 2241 unreg_rsa: 2242 hpre_unregister_rsa(qm); 2243 unlock: 2244 mutex_unlock(&hpre_algs_lock); 2245 return ret; 2246 } 2247 2248 void hpre_algs_unregister(struct hisi_qm *qm) 2249 { 2250 mutex_lock(&hpre_algs_lock); 2251 if (--hpre_available_devs) 2252 goto unlock; 2253 2254 hpre_unregister_x25519(qm); 2255 hpre_unregister_ecdh(qm); 2256 hpre_unregister_dh(qm); 2257 hpre_unregister_rsa(qm); 2258 2259 unlock: 2260 mutex_unlock(&hpre_algs_lock); 2261 } 2262