1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/module.h> 4 #include <crypto/internal/rsa.h> 5 #include <crypto/internal/akcipher.h> 6 #include <crypto/akcipher.h> 7 #include <crypto/kpp.h> 8 #include <crypto/internal/kpp.h> 9 #include <crypto/dh.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/fips.h> 12 #include <crypto/scatterwalk.h> 13 #include "icp_qat_fw_pke.h" 14 #include "adf_accel_devices.h" 15 #include "qat_algs_send.h" 16 #include "adf_transport.h" 17 #include "adf_common_drv.h" 18 #include "qat_crypto.h" 19 20 static DEFINE_MUTEX(algs_lock); 21 static unsigned int active_devs; 22 23 struct qat_rsa_input_params { 24 union { 25 struct { 26 dma_addr_t m; 27 dma_addr_t e; 28 dma_addr_t n; 29 } enc; 30 struct { 31 dma_addr_t c; 32 dma_addr_t d; 33 dma_addr_t n; 34 } dec; 35 struct { 36 dma_addr_t c; 37 dma_addr_t p; 38 dma_addr_t q; 39 dma_addr_t dp; 40 dma_addr_t dq; 41 dma_addr_t qinv; 42 } dec_crt; 43 u64 in_tab[8]; 44 }; 45 } __packed __aligned(64); 46 47 struct qat_rsa_output_params { 48 union { 49 struct { 50 dma_addr_t c; 51 } enc; 52 struct { 53 dma_addr_t m; 54 } dec; 55 u64 out_tab[8]; 56 }; 57 } __packed __aligned(64); 58 59 struct qat_rsa_ctx { 60 char *n; 61 char *e; 62 char *d; 63 char *p; 64 char *q; 65 char *dp; 66 char *dq; 67 char *qinv; 68 dma_addr_t dma_n; 69 dma_addr_t dma_e; 70 dma_addr_t dma_d; 71 dma_addr_t dma_p; 72 dma_addr_t dma_q; 73 dma_addr_t dma_dp; 74 dma_addr_t dma_dq; 75 dma_addr_t dma_qinv; 76 unsigned int key_sz; 77 bool crt_mode; 78 struct qat_crypto_instance *inst; 79 } __packed __aligned(64); 80 81 struct qat_dh_input_params { 82 union { 83 struct { 84 dma_addr_t b; 85 dma_addr_t xa; 86 dma_addr_t p; 87 } in; 88 struct { 89 dma_addr_t xa; 90 dma_addr_t p; 91 } in_g2; 92 u64 in_tab[8]; 93 }; 94 } __packed __aligned(64); 95 96 struct qat_dh_output_params { 97 union { 98 dma_addr_t r; 99 u64 out_tab[8]; 100 }; 101 } __packed __aligned(64); 102 103 struct qat_dh_ctx { 104 char *g; 105 char *xa; 106 char *p; 107 dma_addr_t dma_g; 108 dma_addr_t dma_xa; 109 dma_addr_t dma_p; 110 unsigned int p_size; 111 bool g2; 112 struct qat_crypto_instance *inst; 113 } __packed __aligned(64); 114 115 struct qat_asym_request { 116 union { 117 struct qat_rsa_input_params rsa; 118 struct qat_dh_input_params dh; 119 } in; 120 union { 121 struct qat_rsa_output_params rsa; 122 struct qat_dh_output_params dh; 123 } out; 124 dma_addr_t phy_in; 125 dma_addr_t phy_out; 126 char *src_align; 127 char *dst_align; 128 struct icp_qat_fw_pke_request req; 129 union { 130 struct qat_rsa_ctx *rsa; 131 struct qat_dh_ctx *dh; 132 } ctx; 133 union { 134 struct akcipher_request *rsa; 135 struct kpp_request *dh; 136 } areq; 137 int err; 138 void (*cb)(struct icp_qat_fw_pke_resp *resp); 139 struct qat_alg_req alg_req; 140 } __aligned(64); 141 142 static int qat_alg_send_asym_message(struct qat_asym_request *qat_req, 143 struct qat_crypto_instance *inst, 144 struct crypto_async_request *base) 145 { 146 struct qat_alg_req *alg_req = &qat_req->alg_req; 147 148 alg_req->fw_req = (u32 *)&qat_req->req; 149 alg_req->tx_ring = inst->pke_tx; 150 alg_req->base = base; 151 alg_req->backlog = &inst->backlog; 152 153 return qat_alg_send_message(alg_req); 154 } 155 156 static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp) 157 { 158 struct qat_asym_request *req = (void *)(__force long)resp->opaque; 159 struct kpp_request *areq = req->areq.dh; 160 struct device *dev = &GET_DEV(req->ctx.dh->inst->accel_dev); 161 int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( 162 resp->pke_resp_hdr.comn_resp_flags); 163 164 err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; 165 166 if (areq->src) { 167 dma_unmap_single(dev, req->in.dh.in.b, req->ctx.dh->p_size, 168 DMA_TO_DEVICE); 169 kfree_sensitive(req->src_align); 170 } 171 172 areq->dst_len = req->ctx.dh->p_size; 173 dma_unmap_single(dev, req->out.dh.r, req->ctx.dh->p_size, 174 DMA_FROM_DEVICE); 175 if (req->dst_align) { 176 scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, 177 areq->dst_len, 1); 178 kfree_sensitive(req->dst_align); 179 } 180 181 dma_unmap_single(dev, req->phy_in, sizeof(struct qat_dh_input_params), 182 DMA_TO_DEVICE); 183 dma_unmap_single(dev, req->phy_out, 184 sizeof(struct qat_dh_output_params), 185 DMA_TO_DEVICE); 186 187 kpp_request_complete(areq, err); 188 } 189 190 #define PKE_DH_1536 0x390c1a49 191 #define PKE_DH_G2_1536 0x2e0b1a3e 192 #define PKE_DH_2048 0x4d0c1a60 193 #define PKE_DH_G2_2048 0x3e0b1a55 194 #define PKE_DH_3072 0x510c1a77 195 #define PKE_DH_G2_3072 0x3a0b1a6c 196 #define PKE_DH_4096 0x690c1a8e 197 #define PKE_DH_G2_4096 0x4a0b1a83 198 199 static unsigned long qat_dh_fn_id(unsigned int len, bool g2) 200 { 201 unsigned int bitslen = len << 3; 202 203 switch (bitslen) { 204 case 1536: 205 return g2 ? PKE_DH_G2_1536 : PKE_DH_1536; 206 case 2048: 207 return g2 ? PKE_DH_G2_2048 : PKE_DH_2048; 208 case 3072: 209 return g2 ? PKE_DH_G2_3072 : PKE_DH_3072; 210 case 4096: 211 return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; 212 default: 213 return 0; 214 } 215 } 216 217 static int qat_dh_compute_value(struct kpp_request *req) 218 { 219 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 220 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); 221 struct qat_crypto_instance *inst = ctx->inst; 222 struct device *dev = &GET_DEV(inst->accel_dev); 223 struct qat_asym_request *qat_req = 224 PTR_ALIGN(kpp_request_ctx(req), 64); 225 struct icp_qat_fw_pke_request *msg = &qat_req->req; 226 gfp_t flags = qat_algs_alloc_flags(&req->base); 227 int n_input_params = 0; 228 u8 *vaddr; 229 int ret; 230 231 if (unlikely(!ctx->xa)) 232 return -EINVAL; 233 234 if (req->dst_len < ctx->p_size) { 235 req->dst_len = ctx->p_size; 236 return -EOVERFLOW; 237 } 238 239 if (req->src_len > ctx->p_size) 240 return -EINVAL; 241 242 memset(msg, '\0', sizeof(*msg)); 243 ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, 244 ICP_QAT_FW_COMN_REQ_FLAG_SET); 245 246 msg->pke_hdr.cd_pars.func_id = qat_dh_fn_id(ctx->p_size, 247 !req->src && ctx->g2); 248 if (unlikely(!msg->pke_hdr.cd_pars.func_id)) 249 return -EINVAL; 250 251 qat_req->cb = qat_dh_cb; 252 qat_req->ctx.dh = ctx; 253 qat_req->areq.dh = req; 254 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; 255 msg->pke_hdr.comn_req_flags = 256 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, 257 QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 258 259 /* 260 * If no source is provided use g as base 261 */ 262 if (req->src) { 263 qat_req->in.dh.in.xa = ctx->dma_xa; 264 qat_req->in.dh.in.p = ctx->dma_p; 265 n_input_params = 3; 266 } else { 267 if (ctx->g2) { 268 qat_req->in.dh.in_g2.xa = ctx->dma_xa; 269 qat_req->in.dh.in_g2.p = ctx->dma_p; 270 n_input_params = 2; 271 } else { 272 qat_req->in.dh.in.b = ctx->dma_g; 273 qat_req->in.dh.in.xa = ctx->dma_xa; 274 qat_req->in.dh.in.p = ctx->dma_p; 275 n_input_params = 3; 276 } 277 } 278 279 ret = -ENOMEM; 280 if (req->src) { 281 /* 282 * src can be of any size in valid range, but HW expects it to 283 * be the same as modulo p so in case it is different we need 284 * to allocate a new buf and copy src data. 285 * In other case we just need to map the user provided buffer. 286 * Also need to make sure that it is in contiguous buffer. 287 */ 288 if (sg_is_last(req->src) && req->src_len == ctx->p_size) { 289 qat_req->src_align = NULL; 290 vaddr = sg_virt(req->src); 291 } else { 292 int shift = ctx->p_size - req->src_len; 293 294 qat_req->src_align = kzalloc(ctx->p_size, flags); 295 if (unlikely(!qat_req->src_align)) 296 return ret; 297 298 scatterwalk_map_and_copy(qat_req->src_align + shift, 299 req->src, 0, req->src_len, 0); 300 301 vaddr = qat_req->src_align; 302 } 303 304 qat_req->in.dh.in.b = dma_map_single(dev, vaddr, ctx->p_size, 305 DMA_TO_DEVICE); 306 if (unlikely(dma_mapping_error(dev, qat_req->in.dh.in.b))) 307 goto unmap_src; 308 } 309 /* 310 * dst can be of any size in valid range, but HW expects it to be the 311 * same as modulo m so in case it is different we need to allocate a 312 * new buf and copy src data. 313 * In other case we just need to map the user provided buffer. 314 * Also need to make sure that it is in contiguous buffer. 315 */ 316 if (sg_is_last(req->dst) && req->dst_len == ctx->p_size) { 317 qat_req->dst_align = NULL; 318 vaddr = sg_virt(req->dst); 319 } else { 320 qat_req->dst_align = kzalloc(ctx->p_size, flags); 321 if (unlikely(!qat_req->dst_align)) 322 goto unmap_src; 323 324 vaddr = qat_req->dst_align; 325 } 326 qat_req->out.dh.r = dma_map_single(dev, vaddr, ctx->p_size, 327 DMA_FROM_DEVICE); 328 if (unlikely(dma_mapping_error(dev, qat_req->out.dh.r))) 329 goto unmap_dst; 330 331 qat_req->in.dh.in_tab[n_input_params] = 0; 332 qat_req->out.dh.out_tab[1] = 0; 333 /* Mapping in.in.b or in.in_g2.xa is the same */ 334 qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh, 335 sizeof(struct qat_dh_input_params), 336 DMA_TO_DEVICE); 337 if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) 338 goto unmap_dst; 339 340 qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh, 341 sizeof(struct qat_dh_output_params), 342 DMA_TO_DEVICE); 343 if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) 344 goto unmap_in_params; 345 346 msg->pke_mid.src_data_addr = qat_req->phy_in; 347 msg->pke_mid.dest_data_addr = qat_req->phy_out; 348 msg->pke_mid.opaque = (u64)(__force long)qat_req; 349 msg->input_param_count = n_input_params; 350 msg->output_param_count = 1; 351 352 ret = qat_alg_send_asym_message(qat_req, inst, &req->base); 353 if (ret == -ENOSPC) 354 goto unmap_all; 355 356 return ret; 357 358 unmap_all: 359 if (!dma_mapping_error(dev, qat_req->phy_out)) 360 dma_unmap_single(dev, qat_req->phy_out, 361 sizeof(struct qat_dh_output_params), 362 DMA_TO_DEVICE); 363 unmap_in_params: 364 if (!dma_mapping_error(dev, qat_req->phy_in)) 365 dma_unmap_single(dev, qat_req->phy_in, 366 sizeof(struct qat_dh_input_params), 367 DMA_TO_DEVICE); 368 unmap_dst: 369 if (!dma_mapping_error(dev, qat_req->out.dh.r)) 370 dma_unmap_single(dev, qat_req->out.dh.r, ctx->p_size, 371 DMA_FROM_DEVICE); 372 kfree_sensitive(qat_req->dst_align); 373 unmap_src: 374 if (req->src) { 375 if (!dma_mapping_error(dev, qat_req->in.dh.in.b)) 376 dma_unmap_single(dev, qat_req->in.dh.in.b, 377 ctx->p_size, 378 DMA_TO_DEVICE); 379 kfree_sensitive(qat_req->src_align); 380 } 381 return ret; 382 } 383 384 static int qat_dh_check_params_length(unsigned int p_len) 385 { 386 switch (p_len) { 387 case 1536: 388 case 2048: 389 case 3072: 390 case 4096: 391 return 0; 392 } 393 return -EINVAL; 394 } 395 396 static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) 397 { 398 struct qat_crypto_instance *inst = ctx->inst; 399 struct device *dev = &GET_DEV(inst->accel_dev); 400 401 if (qat_dh_check_params_length(params->p_size << 3)) 402 return -EINVAL; 403 404 ctx->p_size = params->p_size; 405 ctx->p = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL); 406 if (!ctx->p) 407 return -ENOMEM; 408 memcpy(ctx->p, params->p, ctx->p_size); 409 410 /* If g equals 2 don't copy it */ 411 if (params->g_size == 1 && *(char *)params->g == 0x02) { 412 ctx->g2 = true; 413 return 0; 414 } 415 416 ctx->g = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); 417 if (!ctx->g) 418 return -ENOMEM; 419 memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, 420 params->g_size); 421 422 return 0; 423 } 424 425 static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx) 426 { 427 if (ctx->g) { 428 memset(ctx->g, 0, ctx->p_size); 429 dma_free_coherent(dev, ctx->p_size, ctx->g, ctx->dma_g); 430 ctx->g = NULL; 431 } 432 if (ctx->xa) { 433 memset(ctx->xa, 0, ctx->p_size); 434 dma_free_coherent(dev, ctx->p_size, ctx->xa, ctx->dma_xa); 435 ctx->xa = NULL; 436 } 437 if (ctx->p) { 438 memset(ctx->p, 0, ctx->p_size); 439 dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); 440 ctx->p = NULL; 441 } 442 ctx->p_size = 0; 443 ctx->g2 = false; 444 } 445 446 static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, 447 unsigned int len) 448 { 449 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); 450 struct device *dev = &GET_DEV(ctx->inst->accel_dev); 451 struct dh params; 452 int ret; 453 454 if (crypto_dh_decode_key(buf, len, ¶ms) < 0) 455 return -EINVAL; 456 457 /* Free old secret if any */ 458 qat_dh_clear_ctx(dev, ctx); 459 460 ret = qat_dh_set_params(ctx, ¶ms); 461 if (ret < 0) 462 goto err_clear_ctx; 463 464 ctx->xa = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_xa, 465 GFP_KERNEL); 466 if (!ctx->xa) { 467 ret = -ENOMEM; 468 goto err_clear_ctx; 469 } 470 memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, 471 params.key_size); 472 473 return 0; 474 475 err_clear_ctx: 476 qat_dh_clear_ctx(dev, ctx); 477 return ret; 478 } 479 480 static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) 481 { 482 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); 483 484 return ctx->p_size; 485 } 486 487 static int qat_dh_init_tfm(struct crypto_kpp *tfm) 488 { 489 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); 490 struct qat_crypto_instance *inst = 491 qat_crypto_get_instance_node(numa_node_id()); 492 493 if (!inst) 494 return -EINVAL; 495 496 kpp_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64); 497 498 ctx->p_size = 0; 499 ctx->g2 = false; 500 ctx->inst = inst; 501 return 0; 502 } 503 504 static void qat_dh_exit_tfm(struct crypto_kpp *tfm) 505 { 506 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); 507 struct device *dev = &GET_DEV(ctx->inst->accel_dev); 508 509 qat_dh_clear_ctx(dev, ctx); 510 qat_crypto_put_instance(ctx->inst); 511 } 512 513 static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) 514 { 515 struct qat_asym_request *req = (void *)(__force long)resp->opaque; 516 struct akcipher_request *areq = req->areq.rsa; 517 struct device *dev = &GET_DEV(req->ctx.rsa->inst->accel_dev); 518 int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( 519 resp->pke_resp_hdr.comn_resp_flags); 520 521 err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; 522 523 dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz, 524 DMA_TO_DEVICE); 525 526 kfree_sensitive(req->src_align); 527 528 areq->dst_len = req->ctx.rsa->key_sz; 529 dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz, 530 DMA_FROM_DEVICE); 531 if (req->dst_align) { 532 scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, 533 areq->dst_len, 1); 534 535 kfree_sensitive(req->dst_align); 536 } 537 538 dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params), 539 DMA_TO_DEVICE); 540 dma_unmap_single(dev, req->phy_out, 541 sizeof(struct qat_rsa_output_params), 542 DMA_TO_DEVICE); 543 544 akcipher_request_complete(areq, err); 545 } 546 547 void qat_alg_asym_callback(void *_resp) 548 { 549 struct icp_qat_fw_pke_resp *resp = _resp; 550 struct qat_asym_request *areq = (void *)(__force long)resp->opaque; 551 struct qat_instance_backlog *backlog = areq->alg_req.backlog; 552 553 areq->cb(resp); 554 555 qat_alg_send_backlog(backlog); 556 } 557 558 #define PKE_RSA_EP_512 0x1c161b21 559 #define PKE_RSA_EP_1024 0x35111bf7 560 #define PKE_RSA_EP_1536 0x4d111cdc 561 #define PKE_RSA_EP_2048 0x6e111dba 562 #define PKE_RSA_EP_3072 0x7d111ea3 563 #define PKE_RSA_EP_4096 0xa5101f7e 564 565 static unsigned long qat_rsa_enc_fn_id(unsigned int len) 566 { 567 unsigned int bitslen = len << 3; 568 569 switch (bitslen) { 570 case 512: 571 return PKE_RSA_EP_512; 572 case 1024: 573 return PKE_RSA_EP_1024; 574 case 1536: 575 return PKE_RSA_EP_1536; 576 case 2048: 577 return PKE_RSA_EP_2048; 578 case 3072: 579 return PKE_RSA_EP_3072; 580 case 4096: 581 return PKE_RSA_EP_4096; 582 default: 583 return 0; 584 } 585 } 586 587 #define PKE_RSA_DP1_512 0x1c161b3c 588 #define PKE_RSA_DP1_1024 0x35111c12 589 #define PKE_RSA_DP1_1536 0x4d111cf7 590 #define PKE_RSA_DP1_2048 0x6e111dda 591 #define PKE_RSA_DP1_3072 0x7d111ebe 592 #define PKE_RSA_DP1_4096 0xa5101f98 593 594 static unsigned long qat_rsa_dec_fn_id(unsigned int len) 595 { 596 unsigned int bitslen = len << 3; 597 598 switch (bitslen) { 599 case 512: 600 return PKE_RSA_DP1_512; 601 case 1024: 602 return PKE_RSA_DP1_1024; 603 case 1536: 604 return PKE_RSA_DP1_1536; 605 case 2048: 606 return PKE_RSA_DP1_2048; 607 case 3072: 608 return PKE_RSA_DP1_3072; 609 case 4096: 610 return PKE_RSA_DP1_4096; 611 default: 612 return 0; 613 } 614 } 615 616 #define PKE_RSA_DP2_512 0x1c131b57 617 #define PKE_RSA_DP2_1024 0x26131c2d 618 #define PKE_RSA_DP2_1536 0x45111d12 619 #define PKE_RSA_DP2_2048 0x59121dfa 620 #define PKE_RSA_DP2_3072 0x81121ed9 621 #define PKE_RSA_DP2_4096 0xb1111fb2 622 623 static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) 624 { 625 unsigned int bitslen = len << 3; 626 627 switch (bitslen) { 628 case 512: 629 return PKE_RSA_DP2_512; 630 case 1024: 631 return PKE_RSA_DP2_1024; 632 case 1536: 633 return PKE_RSA_DP2_1536; 634 case 2048: 635 return PKE_RSA_DP2_2048; 636 case 3072: 637 return PKE_RSA_DP2_3072; 638 case 4096: 639 return PKE_RSA_DP2_4096; 640 default: 641 return 0; 642 } 643 } 644 645 static int qat_rsa_enc(struct akcipher_request *req) 646 { 647 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 648 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); 649 struct qat_crypto_instance *inst = ctx->inst; 650 struct device *dev = &GET_DEV(inst->accel_dev); 651 struct qat_asym_request *qat_req = 652 PTR_ALIGN(akcipher_request_ctx(req), 64); 653 struct icp_qat_fw_pke_request *msg = &qat_req->req; 654 gfp_t flags = qat_algs_alloc_flags(&req->base); 655 u8 *vaddr; 656 int ret; 657 658 if (unlikely(!ctx->n || !ctx->e)) 659 return -EINVAL; 660 661 if (req->dst_len < ctx->key_sz) { 662 req->dst_len = ctx->key_sz; 663 return -EOVERFLOW; 664 } 665 666 if (req->src_len > ctx->key_sz) 667 return -EINVAL; 668 669 memset(msg, '\0', sizeof(*msg)); 670 ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, 671 ICP_QAT_FW_COMN_REQ_FLAG_SET); 672 msg->pke_hdr.cd_pars.func_id = qat_rsa_enc_fn_id(ctx->key_sz); 673 if (unlikely(!msg->pke_hdr.cd_pars.func_id)) 674 return -EINVAL; 675 676 qat_req->cb = qat_rsa_cb; 677 qat_req->ctx.rsa = ctx; 678 qat_req->areq.rsa = req; 679 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; 680 msg->pke_hdr.comn_req_flags = 681 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, 682 QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 683 684 qat_req->in.rsa.enc.e = ctx->dma_e; 685 qat_req->in.rsa.enc.n = ctx->dma_n; 686 ret = -ENOMEM; 687 688 /* 689 * src can be of any size in valid range, but HW expects it to be the 690 * same as modulo n so in case it is different we need to allocate a 691 * new buf and copy src data. 692 * In other case we just need to map the user provided buffer. 693 * Also need to make sure that it is in contiguous buffer. 694 */ 695 if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { 696 qat_req->src_align = NULL; 697 vaddr = sg_virt(req->src); 698 } else { 699 int shift = ctx->key_sz - req->src_len; 700 701 qat_req->src_align = kzalloc(ctx->key_sz, flags); 702 if (unlikely(!qat_req->src_align)) 703 return ret; 704 705 scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, 706 0, req->src_len, 0); 707 vaddr = qat_req->src_align; 708 } 709 710 qat_req->in.rsa.enc.m = dma_map_single(dev, vaddr, ctx->key_sz, 711 DMA_TO_DEVICE); 712 if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m))) 713 goto unmap_src; 714 715 if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { 716 qat_req->dst_align = NULL; 717 vaddr = sg_virt(req->dst); 718 } else { 719 qat_req->dst_align = kzalloc(ctx->key_sz, flags); 720 if (unlikely(!qat_req->dst_align)) 721 goto unmap_src; 722 vaddr = qat_req->dst_align; 723 } 724 725 qat_req->out.rsa.enc.c = dma_map_single(dev, vaddr, ctx->key_sz, 726 DMA_FROM_DEVICE); 727 if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c))) 728 goto unmap_dst; 729 730 qat_req->in.rsa.in_tab[3] = 0; 731 qat_req->out.rsa.out_tab[1] = 0; 732 qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, 733 sizeof(struct qat_rsa_input_params), 734 DMA_TO_DEVICE); 735 if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) 736 goto unmap_dst; 737 738 qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, 739 sizeof(struct qat_rsa_output_params), 740 DMA_TO_DEVICE); 741 if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) 742 goto unmap_in_params; 743 744 msg->pke_mid.src_data_addr = qat_req->phy_in; 745 msg->pke_mid.dest_data_addr = qat_req->phy_out; 746 msg->pke_mid.opaque = (u64)(__force long)qat_req; 747 msg->input_param_count = 3; 748 msg->output_param_count = 1; 749 750 ret = qat_alg_send_asym_message(qat_req, inst, &req->base); 751 if (ret == -ENOSPC) 752 goto unmap_all; 753 754 return ret; 755 756 unmap_all: 757 if (!dma_mapping_error(dev, qat_req->phy_out)) 758 dma_unmap_single(dev, qat_req->phy_out, 759 sizeof(struct qat_rsa_output_params), 760 DMA_TO_DEVICE); 761 unmap_in_params: 762 if (!dma_mapping_error(dev, qat_req->phy_in)) 763 dma_unmap_single(dev, qat_req->phy_in, 764 sizeof(struct qat_rsa_input_params), 765 DMA_TO_DEVICE); 766 unmap_dst: 767 if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c)) 768 dma_unmap_single(dev, qat_req->out.rsa.enc.c, 769 ctx->key_sz, DMA_FROM_DEVICE); 770 kfree_sensitive(qat_req->dst_align); 771 unmap_src: 772 if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m)) 773 dma_unmap_single(dev, qat_req->in.rsa.enc.m, ctx->key_sz, 774 DMA_TO_DEVICE); 775 kfree_sensitive(qat_req->src_align); 776 return ret; 777 } 778 779 static int qat_rsa_dec(struct akcipher_request *req) 780 { 781 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 782 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); 783 struct qat_crypto_instance *inst = ctx->inst; 784 struct device *dev = &GET_DEV(inst->accel_dev); 785 struct qat_asym_request *qat_req = 786 PTR_ALIGN(akcipher_request_ctx(req), 64); 787 struct icp_qat_fw_pke_request *msg = &qat_req->req; 788 gfp_t flags = qat_algs_alloc_flags(&req->base); 789 u8 *vaddr; 790 int ret; 791 792 if (unlikely(!ctx->n || !ctx->d)) 793 return -EINVAL; 794 795 if (req->dst_len < ctx->key_sz) { 796 req->dst_len = ctx->key_sz; 797 return -EOVERFLOW; 798 } 799 800 if (req->src_len > ctx->key_sz) 801 return -EINVAL; 802 803 memset(msg, '\0', sizeof(*msg)); 804 ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, 805 ICP_QAT_FW_COMN_REQ_FLAG_SET); 806 msg->pke_hdr.cd_pars.func_id = ctx->crt_mode ? 807 qat_rsa_dec_fn_id_crt(ctx->key_sz) : 808 qat_rsa_dec_fn_id(ctx->key_sz); 809 if (unlikely(!msg->pke_hdr.cd_pars.func_id)) 810 return -EINVAL; 811 812 qat_req->cb = qat_rsa_cb; 813 qat_req->ctx.rsa = ctx; 814 qat_req->areq.rsa = req; 815 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; 816 msg->pke_hdr.comn_req_flags = 817 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, 818 QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 819 820 if (ctx->crt_mode) { 821 qat_req->in.rsa.dec_crt.p = ctx->dma_p; 822 qat_req->in.rsa.dec_crt.q = ctx->dma_q; 823 qat_req->in.rsa.dec_crt.dp = ctx->dma_dp; 824 qat_req->in.rsa.dec_crt.dq = ctx->dma_dq; 825 qat_req->in.rsa.dec_crt.qinv = ctx->dma_qinv; 826 } else { 827 qat_req->in.rsa.dec.d = ctx->dma_d; 828 qat_req->in.rsa.dec.n = ctx->dma_n; 829 } 830 ret = -ENOMEM; 831 832 /* 833 * src can be of any size in valid range, but HW expects it to be the 834 * same as modulo n so in case it is different we need to allocate a 835 * new buf and copy src data. 836 * In other case we just need to map the user provided buffer. 837 * Also need to make sure that it is in contiguous buffer. 838 */ 839 if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { 840 qat_req->src_align = NULL; 841 vaddr = sg_virt(req->src); 842 } else { 843 int shift = ctx->key_sz - req->src_len; 844 845 qat_req->src_align = kzalloc(ctx->key_sz, flags); 846 if (unlikely(!qat_req->src_align)) 847 return ret; 848 849 scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, 850 0, req->src_len, 0); 851 vaddr = qat_req->src_align; 852 } 853 854 qat_req->in.rsa.dec.c = dma_map_single(dev, vaddr, ctx->key_sz, 855 DMA_TO_DEVICE); 856 if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c))) 857 goto unmap_src; 858 859 if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { 860 qat_req->dst_align = NULL; 861 vaddr = sg_virt(req->dst); 862 } else { 863 qat_req->dst_align = kzalloc(ctx->key_sz, flags); 864 if (unlikely(!qat_req->dst_align)) 865 goto unmap_src; 866 vaddr = qat_req->dst_align; 867 } 868 qat_req->out.rsa.dec.m = dma_map_single(dev, vaddr, ctx->key_sz, 869 DMA_FROM_DEVICE); 870 if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m))) 871 goto unmap_dst; 872 873 if (ctx->crt_mode) 874 qat_req->in.rsa.in_tab[6] = 0; 875 else 876 qat_req->in.rsa.in_tab[3] = 0; 877 qat_req->out.rsa.out_tab[1] = 0; 878 qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, 879 sizeof(struct qat_rsa_input_params), 880 DMA_TO_DEVICE); 881 if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) 882 goto unmap_dst; 883 884 qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, 885 sizeof(struct qat_rsa_output_params), 886 DMA_TO_DEVICE); 887 if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) 888 goto unmap_in_params; 889 890 msg->pke_mid.src_data_addr = qat_req->phy_in; 891 msg->pke_mid.dest_data_addr = qat_req->phy_out; 892 msg->pke_mid.opaque = (u64)(__force long)qat_req; 893 if (ctx->crt_mode) 894 msg->input_param_count = 6; 895 else 896 msg->input_param_count = 3; 897 898 msg->output_param_count = 1; 899 900 ret = qat_alg_send_asym_message(qat_req, inst, &req->base); 901 if (ret == -ENOSPC) 902 goto unmap_all; 903 904 return ret; 905 906 unmap_all: 907 if (!dma_mapping_error(dev, qat_req->phy_out)) 908 dma_unmap_single(dev, qat_req->phy_out, 909 sizeof(struct qat_rsa_output_params), 910 DMA_TO_DEVICE); 911 unmap_in_params: 912 if (!dma_mapping_error(dev, qat_req->phy_in)) 913 dma_unmap_single(dev, qat_req->phy_in, 914 sizeof(struct qat_rsa_input_params), 915 DMA_TO_DEVICE); 916 unmap_dst: 917 if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m)) 918 dma_unmap_single(dev, qat_req->out.rsa.dec.m, 919 ctx->key_sz, DMA_FROM_DEVICE); 920 kfree_sensitive(qat_req->dst_align); 921 unmap_src: 922 if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c)) 923 dma_unmap_single(dev, qat_req->in.rsa.dec.c, ctx->key_sz, 924 DMA_TO_DEVICE); 925 kfree_sensitive(qat_req->src_align); 926 return ret; 927 } 928 929 static int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value, 930 size_t vlen) 931 { 932 struct qat_crypto_instance *inst = ctx->inst; 933 struct device *dev = &GET_DEV(inst->accel_dev); 934 const char *ptr = value; 935 int ret; 936 937 while (!*ptr && vlen) { 938 ptr++; 939 vlen--; 940 } 941 942 ctx->key_sz = vlen; 943 ret = -EINVAL; 944 /* invalid key size provided */ 945 if (!qat_rsa_enc_fn_id(ctx->key_sz)) 946 goto err; 947 948 ret = -ENOMEM; 949 ctx->n = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_n, GFP_KERNEL); 950 if (!ctx->n) 951 goto err; 952 953 memcpy(ctx->n, ptr, ctx->key_sz); 954 return 0; 955 err: 956 ctx->key_sz = 0; 957 ctx->n = NULL; 958 return ret; 959 } 960 961 static int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value, 962 size_t vlen) 963 { 964 struct qat_crypto_instance *inst = ctx->inst; 965 struct device *dev = &GET_DEV(inst->accel_dev); 966 const char *ptr = value; 967 968 while (!*ptr && vlen) { 969 ptr++; 970 vlen--; 971 } 972 973 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) { 974 ctx->e = NULL; 975 return -EINVAL; 976 } 977 978 ctx->e = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); 979 if (!ctx->e) 980 return -ENOMEM; 981 982 memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); 983 return 0; 984 } 985 986 static int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value, 987 size_t vlen) 988 { 989 struct qat_crypto_instance *inst = ctx->inst; 990 struct device *dev = &GET_DEV(inst->accel_dev); 991 const char *ptr = value; 992 int ret; 993 994 while (!*ptr && vlen) { 995 ptr++; 996 vlen--; 997 } 998 999 ret = -EINVAL; 1000 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 1001 goto err; 1002 1003 ret = -ENOMEM; 1004 ctx->d = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); 1005 if (!ctx->d) 1006 goto err; 1007 1008 memcpy(ctx->d + (ctx->key_sz - vlen), ptr, vlen); 1009 return 0; 1010 err: 1011 ctx->d = NULL; 1012 return ret; 1013 } 1014 1015 static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len) 1016 { 1017 while (!**ptr && *len) { 1018 (*ptr)++; 1019 (*len)--; 1020 } 1021 } 1022 1023 static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) 1024 { 1025 struct qat_crypto_instance *inst = ctx->inst; 1026 struct device *dev = &GET_DEV(inst->accel_dev); 1027 const char *ptr; 1028 unsigned int len; 1029 unsigned int half_key_sz = ctx->key_sz / 2; 1030 1031 /* p */ 1032 ptr = rsa_key->p; 1033 len = rsa_key->p_sz; 1034 qat_rsa_drop_leading_zeros(&ptr, &len); 1035 if (!len) 1036 goto err; 1037 ctx->p = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); 1038 if (!ctx->p) 1039 goto err; 1040 memcpy(ctx->p + (half_key_sz - len), ptr, len); 1041 1042 /* q */ 1043 ptr = rsa_key->q; 1044 len = rsa_key->q_sz; 1045 qat_rsa_drop_leading_zeros(&ptr, &len); 1046 if (!len) 1047 goto free_p; 1048 ctx->q = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); 1049 if (!ctx->q) 1050 goto free_p; 1051 memcpy(ctx->q + (half_key_sz - len), ptr, len); 1052 1053 /* dp */ 1054 ptr = rsa_key->dp; 1055 len = rsa_key->dp_sz; 1056 qat_rsa_drop_leading_zeros(&ptr, &len); 1057 if (!len) 1058 goto free_q; 1059 ctx->dp = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dp, 1060 GFP_KERNEL); 1061 if (!ctx->dp) 1062 goto free_q; 1063 memcpy(ctx->dp + (half_key_sz - len), ptr, len); 1064 1065 /* dq */ 1066 ptr = rsa_key->dq; 1067 len = rsa_key->dq_sz; 1068 qat_rsa_drop_leading_zeros(&ptr, &len); 1069 if (!len) 1070 goto free_dp; 1071 ctx->dq = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dq, 1072 GFP_KERNEL); 1073 if (!ctx->dq) 1074 goto free_dp; 1075 memcpy(ctx->dq + (half_key_sz - len), ptr, len); 1076 1077 /* qinv */ 1078 ptr = rsa_key->qinv; 1079 len = rsa_key->qinv_sz; 1080 qat_rsa_drop_leading_zeros(&ptr, &len); 1081 if (!len) 1082 goto free_dq; 1083 ctx->qinv = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_qinv, 1084 GFP_KERNEL); 1085 if (!ctx->qinv) 1086 goto free_dq; 1087 memcpy(ctx->qinv + (half_key_sz - len), ptr, len); 1088 1089 ctx->crt_mode = true; 1090 return; 1091 1092 free_dq: 1093 memset(ctx->dq, '\0', half_key_sz); 1094 dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); 1095 ctx->dq = NULL; 1096 free_dp: 1097 memset(ctx->dp, '\0', half_key_sz); 1098 dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); 1099 ctx->dp = NULL; 1100 free_q: 1101 memset(ctx->q, '\0', half_key_sz); 1102 dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); 1103 ctx->q = NULL; 1104 free_p: 1105 memset(ctx->p, '\0', half_key_sz); 1106 dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); 1107 ctx->p = NULL; 1108 err: 1109 ctx->crt_mode = false; 1110 } 1111 1112 static void qat_rsa_clear_ctx(struct device *dev, struct qat_rsa_ctx *ctx) 1113 { 1114 unsigned int half_key_sz = ctx->key_sz / 2; 1115 1116 /* Free the old key if any */ 1117 if (ctx->n) 1118 dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); 1119 if (ctx->e) 1120 dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); 1121 if (ctx->d) { 1122 memset(ctx->d, '\0', ctx->key_sz); 1123 dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); 1124 } 1125 if (ctx->p) { 1126 memset(ctx->p, '\0', half_key_sz); 1127 dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); 1128 } 1129 if (ctx->q) { 1130 memset(ctx->q, '\0', half_key_sz); 1131 dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); 1132 } 1133 if (ctx->dp) { 1134 memset(ctx->dp, '\0', half_key_sz); 1135 dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); 1136 } 1137 if (ctx->dq) { 1138 memset(ctx->dq, '\0', half_key_sz); 1139 dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); 1140 } 1141 if (ctx->qinv) { 1142 memset(ctx->qinv, '\0', half_key_sz); 1143 dma_free_coherent(dev, half_key_sz, ctx->qinv, ctx->dma_qinv); 1144 } 1145 1146 ctx->n = NULL; 1147 ctx->e = NULL; 1148 ctx->d = NULL; 1149 ctx->p = NULL; 1150 ctx->q = NULL; 1151 ctx->dp = NULL; 1152 ctx->dq = NULL; 1153 ctx->qinv = NULL; 1154 ctx->crt_mode = false; 1155 ctx->key_sz = 0; 1156 } 1157 1158 static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, 1159 unsigned int keylen, bool private) 1160 { 1161 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); 1162 struct device *dev = &GET_DEV(ctx->inst->accel_dev); 1163 struct rsa_key rsa_key; 1164 int ret; 1165 1166 qat_rsa_clear_ctx(dev, ctx); 1167 1168 if (private) 1169 ret = rsa_parse_priv_key(&rsa_key, key, keylen); 1170 else 1171 ret = rsa_parse_pub_key(&rsa_key, key, keylen); 1172 if (ret < 0) 1173 goto free; 1174 1175 ret = qat_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz); 1176 if (ret < 0) 1177 goto free; 1178 ret = qat_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); 1179 if (ret < 0) 1180 goto free; 1181 if (private) { 1182 ret = qat_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); 1183 if (ret < 0) 1184 goto free; 1185 qat_rsa_setkey_crt(ctx, &rsa_key); 1186 } 1187 1188 if (!ctx->n || !ctx->e) { 1189 /* invalid key provided */ 1190 ret = -EINVAL; 1191 goto free; 1192 } 1193 if (private && !ctx->d) { 1194 /* invalid private key provided */ 1195 ret = -EINVAL; 1196 goto free; 1197 } 1198 1199 return 0; 1200 free: 1201 qat_rsa_clear_ctx(dev, ctx); 1202 return ret; 1203 } 1204 1205 static int qat_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, 1206 unsigned int keylen) 1207 { 1208 return qat_rsa_setkey(tfm, key, keylen, false); 1209 } 1210 1211 static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, 1212 unsigned int keylen) 1213 { 1214 return qat_rsa_setkey(tfm, key, keylen, true); 1215 } 1216 1217 static unsigned int qat_rsa_max_size(struct crypto_akcipher *tfm) 1218 { 1219 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); 1220 1221 return ctx->key_sz; 1222 } 1223 1224 static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) 1225 { 1226 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); 1227 struct qat_crypto_instance *inst = 1228 qat_crypto_get_instance_node(numa_node_id()); 1229 1230 if (!inst) 1231 return -EINVAL; 1232 1233 akcipher_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64); 1234 1235 ctx->key_sz = 0; 1236 ctx->inst = inst; 1237 return 0; 1238 } 1239 1240 static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm) 1241 { 1242 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); 1243 struct device *dev = &GET_DEV(ctx->inst->accel_dev); 1244 1245 qat_rsa_clear_ctx(dev, ctx); 1246 qat_crypto_put_instance(ctx->inst); 1247 } 1248 1249 static struct akcipher_alg rsa = { 1250 .encrypt = qat_rsa_enc, 1251 .decrypt = qat_rsa_dec, 1252 .set_pub_key = qat_rsa_setpubkey, 1253 .set_priv_key = qat_rsa_setprivkey, 1254 .max_size = qat_rsa_max_size, 1255 .init = qat_rsa_init_tfm, 1256 .exit = qat_rsa_exit_tfm, 1257 .base = { 1258 .cra_name = "rsa", 1259 .cra_driver_name = "qat-rsa", 1260 .cra_priority = 1000, 1261 .cra_module = THIS_MODULE, 1262 .cra_ctxsize = sizeof(struct qat_rsa_ctx), 1263 }, 1264 }; 1265 1266 static struct kpp_alg dh = { 1267 .set_secret = qat_dh_set_secret, 1268 .generate_public_key = qat_dh_compute_value, 1269 .compute_shared_secret = qat_dh_compute_value, 1270 .max_size = qat_dh_max_size, 1271 .init = qat_dh_init_tfm, 1272 .exit = qat_dh_exit_tfm, 1273 .base = { 1274 .cra_name = "dh", 1275 .cra_driver_name = "qat-dh", 1276 .cra_priority = 1000, 1277 .cra_module = THIS_MODULE, 1278 .cra_ctxsize = sizeof(struct qat_dh_ctx), 1279 }, 1280 }; 1281 1282 int qat_asym_algs_register(void) 1283 { 1284 int ret = 0; 1285 1286 mutex_lock(&algs_lock); 1287 if (++active_devs == 1) { 1288 rsa.base.cra_flags = 0; 1289 ret = crypto_register_akcipher(&rsa); 1290 if (ret) 1291 goto unlock; 1292 ret = crypto_register_kpp(&dh); 1293 } 1294 unlock: 1295 mutex_unlock(&algs_lock); 1296 return ret; 1297 } 1298 1299 void qat_asym_algs_unregister(void) 1300 { 1301 mutex_lock(&algs_lock); 1302 if (--active_devs == 0) { 1303 crypto_unregister_akcipher(&rsa); 1304 crypto_unregister_kpp(&dh); 1305 } 1306 mutex_unlock(&algs_lock); 1307 } 1308