1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/akcipher.h> 4 #include <crypto/dh.h> 5 #include <crypto/ecc_curve.h> 6 #include <crypto/ecdh.h> 7 #include <crypto/rng.h> 8 #include <crypto/internal/akcipher.h> 9 #include <crypto/internal/kpp.h> 10 #include <crypto/internal/rsa.h> 11 #include <crypto/kpp.h> 12 #include <crypto/scatterwalk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/fips.h> 15 #include <linux/module.h> 16 #include <linux/time.h> 17 #include "hpre.h" 18 19 struct hpre_ctx; 20 21 #define HPRE_CRYPTO_ALG_PRI 1000 22 #define HPRE_ALIGN_SZ 64 23 #define HPRE_BITS_2_BYTES_SHIFT 3 24 #define HPRE_RSA_512BITS_KSZ 64 25 #define HPRE_RSA_1536BITS_KSZ 192 26 #define HPRE_CRT_PRMS 5 27 #define HPRE_CRT_Q 2 28 #define HPRE_CRT_P 3 29 #define HPRE_CRT_INV 4 30 #define HPRE_DH_G_FLAG 0x02 31 #define HPRE_TRY_SEND_TIMES 100 32 #define HPRE_INVLD_REQ_ID (-1) 33 34 #define HPRE_SQE_ALG_BITS 5 35 #define HPRE_SQE_DONE_SHIFT 30 36 #define HPRE_DH_MAX_P_SZ 512 37 38 #define HPRE_DFX_SEC_TO_US 1000000 39 #define HPRE_DFX_US_TO_NS 1000 40 41 #define HPRE_ENABLE_HPCORE_SHIFT 7 42 43 /* due to nist p521 */ 44 #define HPRE_ECC_MAX_KSZ 66 45 46 /* size in bytes of the n prime */ 47 #define HPRE_ECC_NIST_P192_N_SIZE 24 48 #define HPRE_ECC_NIST_P256_N_SIZE 32 49 #define HPRE_ECC_NIST_P384_N_SIZE 48 50 51 /* size in bytes */ 52 #define HPRE_ECC_HW256_KSZ_B 32 53 #define HPRE_ECC_HW384_KSZ_B 48 54 55 /* capability register mask of driver */ 56 #define HPRE_DRV_RSA_MASK_CAP BIT(0) 57 #define HPRE_DRV_DH_MASK_CAP BIT(1) 58 #define HPRE_DRV_ECDH_MASK_CAP BIT(2) 59 #define HPRE_DRV_X25519_MASK_CAP BIT(5) 60 61 static DEFINE_MUTEX(hpre_algs_lock); 62 static unsigned int hpre_available_devs; 63 64 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); 65 66 struct hpre_rsa_ctx { 67 /* low address: e--->n */ 68 char *pubkey; 69 dma_addr_t dma_pubkey; 70 71 /* low address: d--->n */ 72 char *prikey; 73 dma_addr_t dma_prikey; 74 75 /* low address: dq->dp->q->p->qinv */ 76 char *crt_prikey; 77 dma_addr_t dma_crt_prikey; 78 79 struct crypto_akcipher *soft_tfm; 80 }; 81 82 struct hpre_dh_ctx { 83 /* 84 * If base is g we compute the public key 85 * ya = g^xa mod p; [RFC2631 sec 2.1.1] 86 * else if base if the counterpart public key we 87 * compute the shared secret 88 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] 89 * low address: d--->n, please refer to Hisilicon HPRE UM 90 */ 91 char *xa_p; 92 dma_addr_t dma_xa_p; 93 94 char *g; /* m */ 95 dma_addr_t dma_g; 96 }; 97 98 struct hpre_ecdh_ctx { 99 /* low address: p->a->k->b */ 100 unsigned char *p; 101 dma_addr_t dma_p; 102 103 /* low address: x->y */ 104 unsigned char *g; 105 dma_addr_t dma_g; 106 }; 107 108 struct hpre_ctx { 109 struct hisi_qp *qp; 110 struct device *dev; 111 struct hpre_asym_request **req_list; 112 struct hpre *hpre; 113 spinlock_t req_lock; 114 unsigned int key_sz; 115 bool crt_g2_mode; 116 struct idr req_idr; 117 union { 118 struct hpre_rsa_ctx rsa; 119 struct hpre_dh_ctx dh; 120 struct hpre_ecdh_ctx ecdh; 121 }; 122 /* for ecc algorithms */ 123 unsigned int curve_id; 124 /* for high performance core */ 125 u8 enable_hpcore; 126 }; 127 128 struct hpre_asym_request { 129 char *src; 130 char *dst; 131 struct hpre_sqe req; 132 struct hpre_ctx *ctx; 133 union { 134 struct akcipher_request *rsa; 135 struct kpp_request *dh; 136 struct kpp_request *ecdh; 137 } areq; 138 int err; 139 int req_id; 140 hpre_cb cb; 141 struct timespec64 req_time; 142 }; 143 144 static inline unsigned int hpre_align_sz(void) 145 { 146 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1; 147 } 148 149 static inline unsigned int hpre_align_pd(void) 150 { 151 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); 152 } 153 154 static int hpre_alloc_req_id(struct hpre_ctx *ctx) 155 { 156 unsigned long flags; 157 int id; 158 159 spin_lock_irqsave(&ctx->req_lock, flags); 160 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); 161 spin_unlock_irqrestore(&ctx->req_lock, flags); 162 163 return id; 164 } 165 166 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) 167 { 168 unsigned long flags; 169 170 spin_lock_irqsave(&ctx->req_lock, flags); 171 idr_remove(&ctx->req_idr, req_id); 172 spin_unlock_irqrestore(&ctx->req_lock, flags); 173 } 174 175 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) 176 { 177 struct hpre_ctx *ctx; 178 struct hpre_dfx *dfx; 179 int id; 180 181 ctx = hpre_req->ctx; 182 id = hpre_alloc_req_id(ctx); 183 if (unlikely(id < 0)) 184 return -EINVAL; 185 186 ctx->req_list[id] = hpre_req; 187 hpre_req->req_id = id; 188 189 dfx = ctx->hpre->debug.dfx; 190 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 191 ktime_get_ts64(&hpre_req->req_time); 192 193 return id; 194 } 195 196 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) 197 { 198 struct hpre_ctx *ctx = hpre_req->ctx; 199 int id = hpre_req->req_id; 200 201 if (hpre_req->req_id >= 0) { 202 hpre_req->req_id = HPRE_INVLD_REQ_ID; 203 ctx->req_list[id] = NULL; 204 hpre_free_req_id(ctx, id); 205 } 206 } 207 208 static struct hisi_qp *hpre_get_qp_and_start(u8 type) 209 { 210 struct hisi_qp *qp; 211 int ret; 212 213 qp = hpre_create_qp(type); 214 if (!qp) { 215 pr_err("Can not create hpre qp!\n"); 216 return ERR_PTR(-ENODEV); 217 } 218 219 ret = hisi_qm_start_qp(qp, 0); 220 if (ret < 0) { 221 hisi_qm_free_qps(&qp, 1); 222 pci_err(qp->qm->pdev, "Can not start qp!\n"); 223 return ERR_PTR(-EINVAL); 224 } 225 226 return qp; 227 } 228 229 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, 230 struct scatterlist *data, unsigned int len, 231 int is_src, dma_addr_t *tmp) 232 { 233 struct device *dev = hpre_req->ctx->dev; 234 enum dma_data_direction dma_dir; 235 236 if (is_src) { 237 hpre_req->src = NULL; 238 dma_dir = DMA_TO_DEVICE; 239 } else { 240 hpre_req->dst = NULL; 241 dma_dir = DMA_FROM_DEVICE; 242 } 243 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); 244 if (unlikely(dma_mapping_error(dev, *tmp))) { 245 dev_err(dev, "dma map data err!\n"); 246 return -ENOMEM; 247 } 248 249 return 0; 250 } 251 252 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, 253 struct scatterlist *data, unsigned int len, 254 int is_src, dma_addr_t *tmp) 255 { 256 struct hpre_ctx *ctx = hpre_req->ctx; 257 struct device *dev = ctx->dev; 258 void *ptr; 259 int shift; 260 261 shift = ctx->key_sz - len; 262 if (unlikely(shift < 0)) 263 return -EINVAL; 264 265 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC); 266 if (unlikely(!ptr)) 267 return -ENOMEM; 268 269 if (is_src) { 270 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0); 271 hpre_req->src = ptr; 272 } else { 273 hpre_req->dst = ptr; 274 } 275 276 return 0; 277 } 278 279 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, 280 struct scatterlist *data, unsigned int len, 281 int is_src, int is_dh) 282 { 283 struct hpre_sqe *msg = &hpre_req->req; 284 struct hpre_ctx *ctx = hpre_req->ctx; 285 dma_addr_t tmp = 0; 286 int ret; 287 288 /* when the data is dh's source, we should format it */ 289 if ((sg_is_last(data) && len == ctx->key_sz) && 290 ((is_dh && !is_src) || !is_dh)) 291 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); 292 else 293 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); 294 295 if (unlikely(ret)) 296 return ret; 297 298 if (is_src) 299 msg->in = cpu_to_le64(tmp); 300 else 301 msg->out = cpu_to_le64(tmp); 302 303 return 0; 304 } 305 306 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, 307 struct hpre_asym_request *req, 308 struct scatterlist *dst, 309 struct scatterlist *src) 310 { 311 struct device *dev = ctx->dev; 312 struct hpre_sqe *sqe = &req->req; 313 dma_addr_t tmp; 314 315 tmp = le64_to_cpu(sqe->in); 316 if (unlikely(dma_mapping_error(dev, tmp))) 317 return; 318 319 if (src) { 320 if (req->src) 321 dma_free_coherent(dev, ctx->key_sz, req->src, tmp); 322 else 323 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); 324 } 325 326 tmp = le64_to_cpu(sqe->out); 327 if (unlikely(dma_mapping_error(dev, tmp))) 328 return; 329 330 if (req->dst) { 331 if (dst) 332 scatterwalk_map_and_copy(req->dst, dst, 0, 333 ctx->key_sz, 1); 334 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp); 335 } else { 336 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE); 337 } 338 } 339 340 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, 341 void **kreq) 342 { 343 struct hpre_asym_request *req; 344 unsigned int err, done, alg; 345 int id; 346 347 #define HPRE_NO_HW_ERR 0 348 #define HPRE_HW_TASK_DONE 3 349 #define HREE_HW_ERR_MASK GENMASK(10, 0) 350 #define HREE_SQE_DONE_MASK GENMASK(1, 0) 351 #define HREE_ALG_TYPE_MASK GENMASK(4, 0) 352 id = (int)le16_to_cpu(sqe->tag); 353 req = ctx->req_list[id]; 354 hpre_rm_req_from_ctx(req); 355 *kreq = req; 356 357 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & 358 HREE_HW_ERR_MASK; 359 360 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & 361 HREE_SQE_DONE_MASK; 362 363 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) 364 return 0; 365 366 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; 367 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", 368 alg, done, err); 369 370 return -EINVAL; 371 } 372 373 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) 374 { 375 struct hpre *hpre; 376 377 if (!ctx || !qp || qlen < 0) 378 return -EINVAL; 379 380 spin_lock_init(&ctx->req_lock); 381 ctx->qp = qp; 382 ctx->dev = &qp->qm->pdev->dev; 383 384 hpre = container_of(ctx->qp->qm, struct hpre, qm); 385 ctx->hpre = hpre; 386 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); 387 if (!ctx->req_list) 388 return -ENOMEM; 389 ctx->key_sz = 0; 390 ctx->crt_g2_mode = false; 391 idr_init(&ctx->req_idr); 392 393 return 0; 394 } 395 396 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) 397 { 398 if (is_clear_all) { 399 idr_destroy(&ctx->req_idr); 400 kfree(ctx->req_list); 401 hisi_qm_free_qps(&ctx->qp, 1); 402 } 403 404 ctx->crt_g2_mode = false; 405 ctx->key_sz = 0; 406 } 407 408 static bool hpre_is_bd_timeout(struct hpre_asym_request *req, 409 u64 overtime_thrhld) 410 { 411 struct timespec64 reply_time; 412 u64 time_use_us; 413 414 ktime_get_ts64(&reply_time); 415 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) * 416 HPRE_DFX_SEC_TO_US + 417 (reply_time.tv_nsec - req->req_time.tv_nsec) / 418 HPRE_DFX_US_TO_NS; 419 420 if (time_use_us <= overtime_thrhld) 421 return false; 422 423 return true; 424 } 425 426 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp) 427 { 428 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 429 struct hpre_asym_request *req; 430 struct kpp_request *areq; 431 u64 overtime_thrhld; 432 int ret; 433 434 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 435 areq = req->areq.dh; 436 areq->dst_len = ctx->key_sz; 437 438 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 439 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 440 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 441 442 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 443 kpp_request_complete(areq, ret); 444 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 445 } 446 447 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) 448 { 449 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 450 struct hpre_asym_request *req; 451 struct akcipher_request *areq; 452 u64 overtime_thrhld; 453 int ret; 454 455 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 456 457 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 458 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 459 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 460 461 areq = req->areq.rsa; 462 areq->dst_len = ctx->key_sz; 463 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 464 akcipher_request_complete(areq, ret); 465 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 466 } 467 468 static void hpre_alg_cb(struct hisi_qp *qp, void *resp) 469 { 470 struct hpre_ctx *ctx = qp->qp_ctx; 471 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 472 struct hpre_sqe *sqe = resp; 473 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; 474 475 if (unlikely(!req)) { 476 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); 477 return; 478 } 479 480 req->cb(ctx, resp); 481 } 482 483 static void hpre_stop_qp_and_put(struct hisi_qp *qp) 484 { 485 hisi_qm_stop_qp(qp); 486 hisi_qm_free_qps(&qp, 1); 487 } 488 489 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) 490 { 491 struct hisi_qp *qp; 492 int ret; 493 494 qp = hpre_get_qp_and_start(type); 495 if (IS_ERR(qp)) 496 return PTR_ERR(qp); 497 498 qp->qp_ctx = ctx; 499 qp->req_cb = hpre_alg_cb; 500 501 ret = hpre_ctx_set(ctx, qp, qp->sq_depth); 502 if (ret) 503 hpre_stop_qp_and_put(qp); 504 505 return ret; 506 } 507 508 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) 509 { 510 struct hpre_asym_request *h_req; 511 struct hpre_sqe *msg; 512 int req_id; 513 void *tmp; 514 515 if (is_rsa) { 516 struct akcipher_request *akreq = req; 517 518 if (akreq->dst_len < ctx->key_sz) { 519 akreq->dst_len = ctx->key_sz; 520 return -EOVERFLOW; 521 } 522 523 tmp = akcipher_request_ctx(akreq); 524 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 525 h_req->cb = hpre_rsa_cb; 526 h_req->areq.rsa = akreq; 527 msg = &h_req->req; 528 memset(msg, 0, sizeof(*msg)); 529 } else { 530 struct kpp_request *kreq = req; 531 532 if (kreq->dst_len < ctx->key_sz) { 533 kreq->dst_len = ctx->key_sz; 534 return -EOVERFLOW; 535 } 536 537 tmp = kpp_request_ctx(kreq); 538 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 539 h_req->cb = hpre_dh_cb; 540 h_req->areq.dh = kreq; 541 msg = &h_req->req; 542 memset(msg, 0, sizeof(*msg)); 543 msg->key = cpu_to_le64(ctx->dh.dma_xa_p); 544 } 545 546 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 547 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 548 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); 549 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 550 h_req->ctx = ctx; 551 552 req_id = hpre_add_req_to_ctx(h_req); 553 if (req_id < 0) 554 return -EBUSY; 555 556 msg->tag = cpu_to_le16((u16)req_id); 557 558 return 0; 559 } 560 561 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) 562 { 563 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 564 int ctr = 0; 565 int ret; 566 567 do { 568 atomic64_inc(&dfx[HPRE_SEND_CNT].value); 569 spin_lock_bh(&ctx->req_lock); 570 ret = hisi_qp_send(ctx->qp, msg); 571 spin_unlock_bh(&ctx->req_lock); 572 if (ret != -EBUSY) 573 break; 574 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); 575 } while (ctr++ < HPRE_TRY_SEND_TIMES); 576 577 if (likely(!ret)) 578 return ret; 579 580 if (ret != -EBUSY) 581 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value); 582 583 return ret; 584 } 585 586 static int hpre_dh_compute_value(struct kpp_request *req) 587 { 588 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 589 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 590 void *tmp = kpp_request_ctx(req); 591 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 592 struct hpre_sqe *msg = &hpre_req->req; 593 int ret; 594 595 ret = hpre_msg_request_set(ctx, req, false); 596 if (unlikely(ret)) 597 return ret; 598 599 if (req->src) { 600 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); 601 if (unlikely(ret)) 602 goto clear_all; 603 } else { 604 msg->in = cpu_to_le64(ctx->dh.dma_g); 605 } 606 607 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); 608 if (unlikely(ret)) 609 goto clear_all; 610 611 if (ctx->crt_g2_mode && !req->src) 612 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2); 613 else 614 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH); 615 616 /* success */ 617 ret = hpre_send(ctx, msg); 618 if (likely(!ret)) 619 return -EINPROGRESS; 620 621 clear_all: 622 hpre_rm_req_from_ctx(hpre_req); 623 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 624 625 return ret; 626 } 627 628 static int hpre_is_dh_params_length_valid(unsigned int key_sz) 629 { 630 #define _HPRE_DH_GRP1 768 631 #define _HPRE_DH_GRP2 1024 632 #define _HPRE_DH_GRP5 1536 633 #define _HPRE_DH_GRP14 2048 634 #define _HPRE_DH_GRP15 3072 635 #define _HPRE_DH_GRP16 4096 636 switch (key_sz) { 637 case _HPRE_DH_GRP1: 638 case _HPRE_DH_GRP2: 639 case _HPRE_DH_GRP5: 640 case _HPRE_DH_GRP14: 641 case _HPRE_DH_GRP15: 642 case _HPRE_DH_GRP16: 643 return 0; 644 default: 645 return -EINVAL; 646 } 647 } 648 649 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) 650 { 651 struct device *dev = ctx->dev; 652 unsigned int sz; 653 654 if (params->p_size > HPRE_DH_MAX_P_SZ) 655 return -EINVAL; 656 657 if (hpre_is_dh_params_length_valid(params->p_size << 658 HPRE_BITS_2_BYTES_SHIFT)) 659 return -EINVAL; 660 661 sz = ctx->key_sz = params->p_size; 662 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, 663 &ctx->dh.dma_xa_p, GFP_KERNEL); 664 if (!ctx->dh.xa_p) 665 return -ENOMEM; 666 667 memcpy(ctx->dh.xa_p + sz, params->p, sz); 668 669 /* If g equals 2 don't copy it */ 670 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) { 671 ctx->crt_g2_mode = true; 672 return 0; 673 } 674 675 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL); 676 if (!ctx->dh.g) { 677 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 678 ctx->dh.dma_xa_p); 679 ctx->dh.xa_p = NULL; 680 return -ENOMEM; 681 } 682 683 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size); 684 685 return 0; 686 } 687 688 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 689 { 690 struct device *dev = ctx->dev; 691 unsigned int sz = ctx->key_sz; 692 693 if (is_clear_all) 694 hisi_qm_stop_qp(ctx->qp); 695 696 if (ctx->dh.g) { 697 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); 698 ctx->dh.g = NULL; 699 } 700 701 if (ctx->dh.xa_p) { 702 memzero_explicit(ctx->dh.xa_p, sz); 703 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 704 ctx->dh.dma_xa_p); 705 ctx->dh.xa_p = NULL; 706 } 707 708 hpre_ctx_clear(ctx, is_clear_all); 709 } 710 711 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, 712 unsigned int len) 713 { 714 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 715 struct dh params; 716 int ret; 717 718 if (crypto_dh_decode_key(buf, len, ¶ms) < 0) 719 return -EINVAL; 720 721 /* Free old secret if any */ 722 hpre_dh_clear_ctx(ctx, false); 723 724 ret = hpre_dh_set_params(ctx, ¶ms); 725 if (ret < 0) 726 goto err_clear_ctx; 727 728 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, 729 params.key_size); 730 731 return 0; 732 733 err_clear_ctx: 734 hpre_dh_clear_ctx(ctx, false); 735 return ret; 736 } 737 738 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) 739 { 740 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 741 742 return ctx->key_sz; 743 } 744 745 static int hpre_dh_init_tfm(struct crypto_kpp *tfm) 746 { 747 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 748 749 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 750 751 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 752 } 753 754 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) 755 { 756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 757 758 hpre_dh_clear_ctx(ctx, true); 759 } 760 761 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) 762 { 763 while (!**ptr && *len) { 764 (*ptr)++; 765 (*len)--; 766 } 767 } 768 769 static bool hpre_rsa_key_size_is_support(unsigned int len) 770 { 771 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT; 772 773 #define _RSA_1024BITS_KEY_WDTH 1024 774 #define _RSA_2048BITS_KEY_WDTH 2048 775 #define _RSA_3072BITS_KEY_WDTH 3072 776 #define _RSA_4096BITS_KEY_WDTH 4096 777 778 switch (bits) { 779 case _RSA_1024BITS_KEY_WDTH: 780 case _RSA_2048BITS_KEY_WDTH: 781 case _RSA_3072BITS_KEY_WDTH: 782 case _RSA_4096BITS_KEY_WDTH: 783 return true; 784 default: 785 return false; 786 } 787 } 788 789 static int hpre_rsa_enc(struct akcipher_request *req) 790 { 791 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 792 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 793 void *tmp = akcipher_request_ctx(req); 794 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 795 struct hpre_sqe *msg = &hpre_req->req; 796 int ret; 797 798 /* For 512 and 1536 bits key size, use soft tfm instead */ 799 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 800 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 801 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 802 ret = crypto_akcipher_encrypt(req); 803 akcipher_request_set_tfm(req, tfm); 804 return ret; 805 } 806 807 if (unlikely(!ctx->rsa.pubkey)) 808 return -EINVAL; 809 810 ret = hpre_msg_request_set(ctx, req, true); 811 if (unlikely(ret)) 812 return ret; 813 814 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); 815 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); 816 817 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 818 if (unlikely(ret)) 819 goto clear_all; 820 821 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 822 if (unlikely(ret)) 823 goto clear_all; 824 825 /* success */ 826 ret = hpre_send(ctx, msg); 827 if (likely(!ret)) 828 return -EINPROGRESS; 829 830 clear_all: 831 hpre_rm_req_from_ctx(hpre_req); 832 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 833 834 return ret; 835 } 836 837 static int hpre_rsa_dec(struct akcipher_request *req) 838 { 839 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 840 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 841 void *tmp = akcipher_request_ctx(req); 842 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 843 struct hpre_sqe *msg = &hpre_req->req; 844 int ret; 845 846 /* For 512 and 1536 bits key size, use soft tfm instead */ 847 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 848 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 849 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 850 ret = crypto_akcipher_decrypt(req); 851 akcipher_request_set_tfm(req, tfm); 852 return ret; 853 } 854 855 if (unlikely(!ctx->rsa.prikey)) 856 return -EINVAL; 857 858 ret = hpre_msg_request_set(ctx, req, true); 859 if (unlikely(ret)) 860 return ret; 861 862 if (ctx->crt_g2_mode) { 863 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); 864 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 865 HPRE_ALG_NC_CRT); 866 } else { 867 msg->key = cpu_to_le64(ctx->rsa.dma_prikey); 868 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 869 HPRE_ALG_NC_NCRT); 870 } 871 872 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 873 if (unlikely(ret)) 874 goto clear_all; 875 876 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 877 if (unlikely(ret)) 878 goto clear_all; 879 880 /* success */ 881 ret = hpre_send(ctx, msg); 882 if (likely(!ret)) 883 return -EINPROGRESS; 884 885 clear_all: 886 hpre_rm_req_from_ctx(hpre_req); 887 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 888 889 return ret; 890 } 891 892 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, 893 size_t vlen, bool private) 894 { 895 const char *ptr = value; 896 897 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 898 899 ctx->key_sz = vlen; 900 901 /* if invalid key size provided, we use software tfm */ 902 if (!hpre_rsa_key_size_is_support(ctx->key_sz)) 903 return 0; 904 905 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, 906 &ctx->rsa.dma_pubkey, 907 GFP_KERNEL); 908 if (!ctx->rsa.pubkey) 909 return -ENOMEM; 910 911 if (private) { 912 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, 913 &ctx->rsa.dma_prikey, 914 GFP_KERNEL); 915 if (!ctx->rsa.prikey) { 916 dma_free_coherent(ctx->dev, vlen << 1, 917 ctx->rsa.pubkey, 918 ctx->rsa.dma_pubkey); 919 ctx->rsa.pubkey = NULL; 920 return -ENOMEM; 921 } 922 memcpy(ctx->rsa.prikey + vlen, ptr, vlen); 923 } 924 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen); 925 926 /* Using hardware HPRE to do RSA */ 927 return 1; 928 } 929 930 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value, 931 size_t vlen) 932 { 933 const char *ptr = value; 934 935 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 936 937 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 938 return -EINVAL; 939 940 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen); 941 942 return 0; 943 } 944 945 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value, 946 size_t vlen) 947 { 948 const char *ptr = value; 949 950 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 951 952 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 953 return -EINVAL; 954 955 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen); 956 957 return 0; 958 } 959 960 static int hpre_crt_para_get(char *para, size_t para_sz, 961 const char *raw, size_t raw_sz) 962 { 963 const char *ptr = raw; 964 size_t len = raw_sz; 965 966 hpre_rsa_drop_leading_zeros(&ptr, &len); 967 if (!len || len > para_sz) 968 return -EINVAL; 969 970 memcpy(para + para_sz - len, ptr, len); 971 972 return 0; 973 } 974 975 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) 976 { 977 unsigned int hlf_ksz = ctx->key_sz >> 1; 978 struct device *dev = ctx->dev; 979 u64 offset; 980 int ret; 981 982 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, 983 &ctx->rsa.dma_crt_prikey, 984 GFP_KERNEL); 985 if (!ctx->rsa.crt_prikey) 986 return -ENOMEM; 987 988 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz, 989 rsa_key->dq, rsa_key->dq_sz); 990 if (ret) 991 goto free_key; 992 993 offset = hlf_ksz; 994 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 995 rsa_key->dp, rsa_key->dp_sz); 996 if (ret) 997 goto free_key; 998 999 offset = hlf_ksz * HPRE_CRT_Q; 1000 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1001 rsa_key->q, rsa_key->q_sz); 1002 if (ret) 1003 goto free_key; 1004 1005 offset = hlf_ksz * HPRE_CRT_P; 1006 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1007 rsa_key->p, rsa_key->p_sz); 1008 if (ret) 1009 goto free_key; 1010 1011 offset = hlf_ksz * HPRE_CRT_INV; 1012 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 1013 rsa_key->qinv, rsa_key->qinv_sz); 1014 if (ret) 1015 goto free_key; 1016 1017 ctx->crt_g2_mode = true; 1018 1019 return 0; 1020 1021 free_key: 1022 offset = hlf_ksz * HPRE_CRT_PRMS; 1023 memzero_explicit(ctx->rsa.crt_prikey, offset); 1024 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey, 1025 ctx->rsa.dma_crt_prikey); 1026 ctx->rsa.crt_prikey = NULL; 1027 ctx->crt_g2_mode = false; 1028 1029 return ret; 1030 } 1031 1032 /* If it is clear all, all the resources of the QP will be cleaned. */ 1033 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 1034 { 1035 unsigned int half_key_sz = ctx->key_sz >> 1; 1036 struct device *dev = ctx->dev; 1037 1038 if (is_clear_all) 1039 hisi_qm_stop_qp(ctx->qp); 1040 1041 if (ctx->rsa.pubkey) { 1042 dma_free_coherent(dev, ctx->key_sz << 1, 1043 ctx->rsa.pubkey, ctx->rsa.dma_pubkey); 1044 ctx->rsa.pubkey = NULL; 1045 } 1046 1047 if (ctx->rsa.crt_prikey) { 1048 memzero_explicit(ctx->rsa.crt_prikey, 1049 half_key_sz * HPRE_CRT_PRMS); 1050 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS, 1051 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey); 1052 ctx->rsa.crt_prikey = NULL; 1053 } 1054 1055 if (ctx->rsa.prikey) { 1056 memzero_explicit(ctx->rsa.prikey, ctx->key_sz); 1057 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey, 1058 ctx->rsa.dma_prikey); 1059 ctx->rsa.prikey = NULL; 1060 } 1061 1062 hpre_ctx_clear(ctx, is_clear_all); 1063 } 1064 1065 /* 1066 * we should judge if it is CRT or not, 1067 * CRT: return true, N-CRT: return false . 1068 */ 1069 static bool hpre_is_crt_key(struct rsa_key *key) 1070 { 1071 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz + 1072 key->qinv_sz; 1073 1074 #define LEN_OF_NCRT_PARA 5 1075 1076 /* N-CRT less than 5 parameters */ 1077 return len > LEN_OF_NCRT_PARA; 1078 } 1079 1080 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, 1081 unsigned int keylen, bool private) 1082 { 1083 struct rsa_key rsa_key; 1084 int ret; 1085 1086 hpre_rsa_clear_ctx(ctx, false); 1087 1088 if (private) 1089 ret = rsa_parse_priv_key(&rsa_key, key, keylen); 1090 else 1091 ret = rsa_parse_pub_key(&rsa_key, key, keylen); 1092 if (ret < 0) 1093 return ret; 1094 1095 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private); 1096 if (ret <= 0) 1097 return ret; 1098 1099 if (private) { 1100 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); 1101 if (ret < 0) 1102 goto free; 1103 1104 if (hpre_is_crt_key(&rsa_key)) { 1105 ret = hpre_rsa_setkey_crt(ctx, &rsa_key); 1106 if (ret < 0) 1107 goto free; 1108 } 1109 } 1110 1111 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); 1112 if (ret < 0) 1113 goto free; 1114 1115 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) { 1116 ret = -EINVAL; 1117 goto free; 1118 } 1119 1120 return 0; 1121 1122 free: 1123 hpre_rsa_clear_ctx(ctx, false); 1124 return ret; 1125 } 1126 1127 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, 1128 unsigned int keylen) 1129 { 1130 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1131 int ret; 1132 1133 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen); 1134 if (ret) 1135 return ret; 1136 1137 return hpre_rsa_setkey(ctx, key, keylen, false); 1138 } 1139 1140 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, 1141 unsigned int keylen) 1142 { 1143 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1144 int ret; 1145 1146 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen); 1147 if (ret) 1148 return ret; 1149 1150 return hpre_rsa_setkey(ctx, key, keylen, true); 1151 } 1152 1153 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) 1154 { 1155 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1156 1157 /* For 512 and 1536 bits key size, use soft tfm instead */ 1158 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 1159 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) 1160 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); 1161 1162 return ctx->key_sz; 1163 } 1164 1165 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) 1166 { 1167 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1168 int ret; 1169 1170 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0); 1171 if (IS_ERR(ctx->rsa.soft_tfm)) { 1172 pr_err("Can not alloc_akcipher!\n"); 1173 return PTR_ERR(ctx->rsa.soft_tfm); 1174 } 1175 1176 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) + 1177 hpre_align_pd()); 1178 1179 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 1180 if (ret) 1181 crypto_free_akcipher(ctx->rsa.soft_tfm); 1182 1183 return ret; 1184 } 1185 1186 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) 1187 { 1188 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1189 1190 hpre_rsa_clear_ctx(ctx, true); 1191 crypto_free_akcipher(ctx->rsa.soft_tfm); 1192 } 1193 1194 static void hpre_key_to_big_end(u8 *data, int len) 1195 { 1196 int i, j; 1197 1198 for (i = 0; i < len / 2; i++) { 1199 j = len - i - 1; 1200 swap(data[j], data[i]); 1201 } 1202 } 1203 1204 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 1205 { 1206 struct device *dev = ctx->dev; 1207 unsigned int sz = ctx->key_sz; 1208 unsigned int shift = sz << 1; 1209 1210 if (is_clear_all) 1211 hisi_qm_stop_qp(ctx->qp); 1212 1213 if (ctx->ecdh.p) { 1214 /* ecdh: p->a->k->b */ 1215 memzero_explicit(ctx->ecdh.p + shift, sz); 1216 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1217 ctx->ecdh.p = NULL; 1218 } 1219 1220 hpre_ctx_clear(ctx, is_clear_all); 1221 } 1222 1223 /* 1224 * The bits of 192/224/256/384/521 are supported by HPRE, 1225 * and convert the bits like: 1226 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576; 1227 * If the parameter bit width is insufficient, then we fill in the 1228 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8; 1229 */ 1230 static unsigned int hpre_ecdh_supported_curve(unsigned short id) 1231 { 1232 switch (id) { 1233 case ECC_CURVE_NIST_P192: 1234 case ECC_CURVE_NIST_P256: 1235 return HPRE_ECC_HW256_KSZ_B; 1236 case ECC_CURVE_NIST_P384: 1237 return HPRE_ECC_HW384_KSZ_B; 1238 default: 1239 break; 1240 } 1241 1242 return 0; 1243 } 1244 1245 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits) 1246 { 1247 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64); 1248 u8 i = 0; 1249 1250 while (i < ndigits - 1) { 1251 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64)); 1252 i++; 1253 } 1254 1255 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz); 1256 hpre_key_to_big_end((u8 *)addr, cur_sz); 1257 } 1258 1259 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params, 1260 unsigned int cur_sz) 1261 { 1262 unsigned int shifta = ctx->key_sz << 1; 1263 unsigned int shiftb = ctx->key_sz << 2; 1264 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz; 1265 void *a = ctx->ecdh.p + shifta - cur_sz; 1266 void *b = ctx->ecdh.p + shiftb - cur_sz; 1267 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz; 1268 void *y = ctx->ecdh.g + shifta - cur_sz; 1269 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id); 1270 char *n; 1271 1272 if (unlikely(!curve)) 1273 return -EINVAL; 1274 1275 n = kzalloc(ctx->key_sz, GFP_KERNEL); 1276 if (!n) 1277 return -ENOMEM; 1278 1279 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits); 1280 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits); 1281 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits); 1282 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits); 1283 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits); 1284 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits); 1285 1286 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) { 1287 kfree(n); 1288 return -EINVAL; 1289 } 1290 1291 kfree(n); 1292 return 0; 1293 } 1294 1295 static unsigned int hpre_ecdh_get_curvesz(unsigned short id) 1296 { 1297 switch (id) { 1298 case ECC_CURVE_NIST_P192: 1299 return HPRE_ECC_NIST_P192_N_SIZE; 1300 case ECC_CURVE_NIST_P256: 1301 return HPRE_ECC_NIST_P256_N_SIZE; 1302 case ECC_CURVE_NIST_P384: 1303 return HPRE_ECC_NIST_P384_N_SIZE; 1304 default: 1305 break; 1306 } 1307 1308 return 0; 1309 } 1310 1311 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) 1312 { 1313 struct device *dev = ctx->dev; 1314 unsigned int sz, shift, curve_sz; 1315 int ret; 1316 1317 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id); 1318 if (!ctx->key_sz) 1319 return -EINVAL; 1320 1321 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1322 if (!curve_sz || params->key_size > curve_sz) 1323 return -EINVAL; 1324 1325 sz = ctx->key_sz; 1326 1327 if (!ctx->ecdh.p) { 1328 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p, 1329 GFP_KERNEL); 1330 if (!ctx->ecdh.p) 1331 return -ENOMEM; 1332 } 1333 1334 shift = sz << 2; 1335 ctx->ecdh.g = ctx->ecdh.p + shift; 1336 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift; 1337 1338 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz); 1339 if (ret) { 1340 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret); 1341 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1342 ctx->ecdh.p = NULL; 1343 return ret; 1344 } 1345 1346 return 0; 1347 } 1348 1349 static bool hpre_key_is_zero(char *key, unsigned short key_sz) 1350 { 1351 int i; 1352 1353 for (i = 0; i < key_sz; i++) 1354 if (key[i]) 1355 return false; 1356 1357 return true; 1358 } 1359 1360 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) 1361 { 1362 struct device *dev = ctx->dev; 1363 int ret; 1364 1365 ret = crypto_get_default_rng(); 1366 if (ret) { 1367 dev_err(dev, "failed to get default rng, ret = %d!\n", ret); 1368 return ret; 1369 } 1370 1371 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key, 1372 params->key_size); 1373 crypto_put_default_rng(); 1374 if (ret) 1375 dev_err(dev, "failed to get rng, ret = %d!\n", ret); 1376 1377 return ret; 1378 } 1379 1380 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, 1381 unsigned int len) 1382 { 1383 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1384 unsigned int sz, sz_shift, curve_sz; 1385 struct device *dev = ctx->dev; 1386 char key[HPRE_ECC_MAX_KSZ]; 1387 struct ecdh params; 1388 int ret; 1389 1390 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { 1391 dev_err(dev, "failed to decode ecdh key!\n"); 1392 return -EINVAL; 1393 } 1394 1395 /* Use stdrng to generate private key */ 1396 if (!params.key || !params.key_size) { 1397 params.key = key; 1398 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1399 if (!curve_sz) { 1400 dev_err(dev, "Invalid curve size!\n"); 1401 return -EINVAL; 1402 } 1403 1404 params.key_size = curve_sz - 1; 1405 ret = ecdh_gen_privkey(ctx, ¶ms); 1406 if (ret) 1407 return ret; 1408 } 1409 1410 if (hpre_key_is_zero(params.key, params.key_size)) { 1411 dev_err(dev, "Invalid hpre key!\n"); 1412 return -EINVAL; 1413 } 1414 1415 hpre_ecc_clear_ctx(ctx, false); 1416 1417 ret = hpre_ecdh_set_param(ctx, ¶ms); 1418 if (ret < 0) { 1419 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret); 1420 return ret; 1421 } 1422 1423 sz = ctx->key_sz; 1424 sz_shift = (sz << 1) + sz - params.key_size; 1425 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size); 1426 1427 return 0; 1428 } 1429 1430 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, 1431 struct hpre_asym_request *req, 1432 struct scatterlist *dst, 1433 struct scatterlist *src) 1434 { 1435 struct device *dev = ctx->dev; 1436 struct hpre_sqe *sqe = &req->req; 1437 dma_addr_t dma; 1438 1439 dma = le64_to_cpu(sqe->in); 1440 if (unlikely(dma_mapping_error(dev, dma))) 1441 return; 1442 1443 if (src && req->src) 1444 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); 1445 1446 dma = le64_to_cpu(sqe->out); 1447 if (unlikely(dma_mapping_error(dev, dma))) 1448 return; 1449 1450 if (req->dst) 1451 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); 1452 if (dst) 1453 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE); 1454 } 1455 1456 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) 1457 { 1458 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1459 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1460 struct hpre_asym_request *req = NULL; 1461 struct kpp_request *areq; 1462 u64 overtime_thrhld; 1463 char *p; 1464 int ret; 1465 1466 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1467 areq = req->areq.ecdh; 1468 areq->dst_len = ctx->key_sz << 1; 1469 1470 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1471 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1472 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1473 1474 /* Do unmap before data processing */ 1475 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1476 1477 p = sg_virt(areq->dst); 1478 memmove(p, p + ctx->key_sz - curve_sz, curve_sz); 1479 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); 1480 1481 kpp_request_complete(areq, ret); 1482 1483 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1484 } 1485 1486 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, 1487 struct kpp_request *req) 1488 { 1489 struct hpre_asym_request *h_req; 1490 struct hpre_sqe *msg; 1491 int req_id; 1492 void *tmp; 1493 1494 if (req->dst_len < ctx->key_sz << 1) { 1495 req->dst_len = ctx->key_sz << 1; 1496 return -EINVAL; 1497 } 1498 1499 tmp = kpp_request_ctx(req); 1500 h_req = PTR_ALIGN(tmp, hpre_align_sz()); 1501 h_req->cb = hpre_ecdh_cb; 1502 h_req->areq.ecdh = req; 1503 msg = &h_req->req; 1504 memset(msg, 0, sizeof(*msg)); 1505 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1506 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1507 msg->key = cpu_to_le64(ctx->ecdh.dma_p); 1508 1509 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1510 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1511 h_req->ctx = ctx; 1512 1513 req_id = hpre_add_req_to_ctx(h_req); 1514 if (req_id < 0) 1515 return -EBUSY; 1516 1517 msg->tag = cpu_to_le16((u16)req_id); 1518 return 0; 1519 } 1520 1521 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, 1522 struct scatterlist *data, unsigned int len) 1523 { 1524 struct hpre_sqe *msg = &hpre_req->req; 1525 struct hpre_ctx *ctx = hpre_req->ctx; 1526 struct device *dev = ctx->dev; 1527 unsigned int tmpshift; 1528 dma_addr_t dma = 0; 1529 void *ptr; 1530 int shift; 1531 1532 /* Src_data include gx and gy. */ 1533 shift = ctx->key_sz - (len >> 1); 1534 if (unlikely(shift < 0)) 1535 return -EINVAL; 1536 1537 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL); 1538 if (unlikely(!ptr)) 1539 return -ENOMEM; 1540 1541 tmpshift = ctx->key_sz << 1; 1542 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0); 1543 memcpy(ptr + shift, ptr + tmpshift, len >> 1); 1544 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1); 1545 1546 hpre_req->src = ptr; 1547 msg->in = cpu_to_le64(dma); 1548 return 0; 1549 } 1550 1551 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, 1552 struct scatterlist *data, unsigned int len) 1553 { 1554 struct hpre_sqe *msg = &hpre_req->req; 1555 struct hpre_ctx *ctx = hpre_req->ctx; 1556 struct device *dev = ctx->dev; 1557 dma_addr_t dma; 1558 1559 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { 1560 dev_err(dev, "data or data length is illegal!\n"); 1561 return -EINVAL; 1562 } 1563 1564 hpre_req->dst = NULL; 1565 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1566 if (unlikely(dma_mapping_error(dev, dma))) { 1567 dev_err(dev, "dma map data err!\n"); 1568 return -ENOMEM; 1569 } 1570 1571 msg->out = cpu_to_le64(dma); 1572 return 0; 1573 } 1574 1575 static int hpre_ecdh_compute_value(struct kpp_request *req) 1576 { 1577 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1578 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1579 struct device *dev = ctx->dev; 1580 void *tmp = kpp_request_ctx(req); 1581 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); 1582 struct hpre_sqe *msg = &hpre_req->req; 1583 int ret; 1584 1585 ret = hpre_ecdh_msg_request_set(ctx, req); 1586 if (unlikely(ret)) { 1587 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret); 1588 return ret; 1589 } 1590 1591 if (req->src) { 1592 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len); 1593 if (unlikely(ret)) { 1594 dev_err(dev, "failed to init src data, ret = %d!\n", ret); 1595 goto clear_all; 1596 } 1597 } else { 1598 msg->in = cpu_to_le64(ctx->ecdh.dma_g); 1599 } 1600 1601 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len); 1602 if (unlikely(ret)) { 1603 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1604 goto clear_all; 1605 } 1606 1607 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); 1608 msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT; 1609 1610 ret = hpre_send(ctx, msg); 1611 if (likely(!ret)) 1612 return -EINPROGRESS; 1613 1614 clear_all: 1615 hpre_rm_req_from_ctx(hpre_req); 1616 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1617 return ret; 1618 } 1619 1620 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) 1621 { 1622 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1623 1624 /* max size is the pub_key_size, include x and y */ 1625 return ctx->key_sz << 1; 1626 } 1627 1628 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) 1629 { 1630 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1631 1632 ctx->curve_id = ECC_CURVE_NIST_P192; 1633 1634 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1635 1636 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1637 } 1638 1639 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) 1640 { 1641 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1642 1643 ctx->curve_id = ECC_CURVE_NIST_P256; 1644 ctx->enable_hpcore = 1; 1645 1646 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1647 1648 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1649 } 1650 1651 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) 1652 { 1653 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1654 1655 ctx->curve_id = ECC_CURVE_NIST_P384; 1656 1657 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1658 1659 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1660 } 1661 1662 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) 1663 { 1664 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1665 1666 hpre_ecc_clear_ctx(ctx, true); 1667 } 1668 1669 static struct akcipher_alg rsa = { 1670 .encrypt = hpre_rsa_enc, 1671 .decrypt = hpre_rsa_dec, 1672 .set_pub_key = hpre_rsa_setpubkey, 1673 .set_priv_key = hpre_rsa_setprivkey, 1674 .max_size = hpre_rsa_max_size, 1675 .init = hpre_rsa_init_tfm, 1676 .exit = hpre_rsa_exit_tfm, 1677 .base = { 1678 .cra_ctxsize = sizeof(struct hpre_ctx), 1679 .cra_priority = HPRE_CRYPTO_ALG_PRI, 1680 .cra_name = "rsa", 1681 .cra_driver_name = "hpre-rsa", 1682 .cra_module = THIS_MODULE, 1683 }, 1684 }; 1685 1686 static struct kpp_alg dh = { 1687 .set_secret = hpre_dh_set_secret, 1688 .generate_public_key = hpre_dh_compute_value, 1689 .compute_shared_secret = hpre_dh_compute_value, 1690 .max_size = hpre_dh_max_size, 1691 .init = hpre_dh_init_tfm, 1692 .exit = hpre_dh_exit_tfm, 1693 .base = { 1694 .cra_ctxsize = sizeof(struct hpre_ctx), 1695 .cra_priority = HPRE_CRYPTO_ALG_PRI, 1696 .cra_name = "dh", 1697 .cra_driver_name = "hpre-dh", 1698 .cra_module = THIS_MODULE, 1699 }, 1700 }; 1701 1702 static struct kpp_alg ecdh_curves[] = { 1703 { 1704 .set_secret = hpre_ecdh_set_secret, 1705 .generate_public_key = hpre_ecdh_compute_value, 1706 .compute_shared_secret = hpre_ecdh_compute_value, 1707 .max_size = hpre_ecdh_max_size, 1708 .init = hpre_ecdh_nist_p192_init_tfm, 1709 .exit = hpre_ecdh_exit_tfm, 1710 .base = { 1711 .cra_ctxsize = sizeof(struct hpre_ctx), 1712 .cra_priority = HPRE_CRYPTO_ALG_PRI, 1713 .cra_name = "ecdh-nist-p192", 1714 .cra_driver_name = "hpre-ecdh-nist-p192", 1715 .cra_module = THIS_MODULE, 1716 }, 1717 }, { 1718 .set_secret = hpre_ecdh_set_secret, 1719 .generate_public_key = hpre_ecdh_compute_value, 1720 .compute_shared_secret = hpre_ecdh_compute_value, 1721 .max_size = hpre_ecdh_max_size, 1722 .init = hpre_ecdh_nist_p256_init_tfm, 1723 .exit = hpre_ecdh_exit_tfm, 1724 .base = { 1725 .cra_ctxsize = sizeof(struct hpre_ctx), 1726 .cra_priority = HPRE_CRYPTO_ALG_PRI, 1727 .cra_name = "ecdh-nist-p256", 1728 .cra_driver_name = "hpre-ecdh-nist-p256", 1729 .cra_module = THIS_MODULE, 1730 }, 1731 }, { 1732 .set_secret = hpre_ecdh_set_secret, 1733 .generate_public_key = hpre_ecdh_compute_value, 1734 .compute_shared_secret = hpre_ecdh_compute_value, 1735 .max_size = hpre_ecdh_max_size, 1736 .init = hpre_ecdh_nist_p384_init_tfm, 1737 .exit = hpre_ecdh_exit_tfm, 1738 .base = { 1739 .cra_ctxsize = sizeof(struct hpre_ctx), 1740 .cra_priority = HPRE_CRYPTO_ALG_PRI, 1741 .cra_name = "ecdh-nist-p384", 1742 .cra_driver_name = "hpre-ecdh-nist-p384", 1743 .cra_module = THIS_MODULE, 1744 }, 1745 } 1746 }; 1747 1748 static int hpre_register_rsa(struct hisi_qm *qm) 1749 { 1750 int ret; 1751 1752 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) 1753 return 0; 1754 1755 rsa.base.cra_flags = 0; 1756 ret = crypto_register_akcipher(&rsa); 1757 if (ret) 1758 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret); 1759 1760 return ret; 1761 } 1762 1763 static void hpre_unregister_rsa(struct hisi_qm *qm) 1764 { 1765 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) 1766 return; 1767 1768 crypto_unregister_akcipher(&rsa); 1769 } 1770 1771 static int hpre_register_dh(struct hisi_qm *qm) 1772 { 1773 int ret; 1774 1775 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) 1776 return 0; 1777 1778 ret = crypto_register_kpp(&dh); 1779 if (ret) 1780 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret); 1781 1782 return ret; 1783 } 1784 1785 static void hpre_unregister_dh(struct hisi_qm *qm) 1786 { 1787 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) 1788 return; 1789 1790 crypto_unregister_kpp(&dh); 1791 } 1792 1793 static int hpre_register_ecdh(struct hisi_qm *qm) 1794 { 1795 int ret, i; 1796 1797 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) 1798 return 0; 1799 1800 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) { 1801 ret = crypto_register_kpp(&ecdh_curves[i]); 1802 if (ret) { 1803 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n", 1804 ecdh_curves[i].base.cra_name, ret); 1805 goto unreg_kpp; 1806 } 1807 } 1808 1809 return 0; 1810 1811 unreg_kpp: 1812 for (--i; i >= 0; --i) 1813 crypto_unregister_kpp(&ecdh_curves[i]); 1814 1815 return ret; 1816 } 1817 1818 static void hpre_unregister_ecdh(struct hisi_qm *qm) 1819 { 1820 int i; 1821 1822 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) 1823 return; 1824 1825 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i) 1826 crypto_unregister_kpp(&ecdh_curves[i]); 1827 } 1828 1829 int hpre_algs_register(struct hisi_qm *qm) 1830 { 1831 int ret = 0; 1832 1833 mutex_lock(&hpre_algs_lock); 1834 if (hpre_available_devs) { 1835 hpre_available_devs++; 1836 goto unlock; 1837 } 1838 1839 ret = hpre_register_rsa(qm); 1840 if (ret) 1841 goto unlock; 1842 1843 ret = hpre_register_dh(qm); 1844 if (ret) 1845 goto unreg_rsa; 1846 1847 ret = hpre_register_ecdh(qm); 1848 if (ret) 1849 goto unreg_dh; 1850 1851 hpre_available_devs++; 1852 mutex_unlock(&hpre_algs_lock); 1853 1854 return ret; 1855 1856 unreg_dh: 1857 hpre_unregister_dh(qm); 1858 unreg_rsa: 1859 hpre_unregister_rsa(qm); 1860 unlock: 1861 mutex_unlock(&hpre_algs_lock); 1862 return ret; 1863 } 1864 1865 void hpre_algs_unregister(struct hisi_qm *qm) 1866 { 1867 mutex_lock(&hpre_algs_lock); 1868 if (--hpre_available_devs) 1869 goto unlock; 1870 1871 hpre_unregister_ecdh(qm); 1872 hpre_unregister_dh(qm); 1873 hpre_unregister_rsa(qm); 1874 1875 unlock: 1876 mutex_unlock(&hpre_algs_lock); 1877 } 1878