1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * StarFive Public Key Algo acceleration driver 4 * 5 * Copyright (c) 2022 StarFive Technology 6 */ 7 8 #include <linux/crypto.h> 9 #include <linux/iopoll.h> 10 #include <crypto/akcipher.h> 11 #include <crypto/algapi.h> 12 #include <crypto/internal/akcipher.h> 13 #include <crypto/internal/rsa.h> 14 #include <crypto/scatterwalk.h> 15 16 #include "jh7110-cryp.h" 17 18 #define STARFIVE_PKA_REGS_OFFSET 0x400 19 #define STARFIVE_PKA_CACR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x0) 20 #define STARFIVE_PKA_CASR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x4) 21 #define STARFIVE_PKA_CAAR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x8) 22 #define STARFIVE_PKA_CAER_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x108) 23 #define STARFIVE_PKA_CANR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x208) 24 25 /* R ^ 2 mod N and N0' */ 26 #define CRYPTO_CMD_PRE 0x0 27 /* A * R mod N ==> A */ 28 #define CRYPTO_CMD_ARN 0x5 29 /* A * E * R mod N ==> A */ 30 #define CRYPTO_CMD_AERN 0x6 31 /* A * A * R mod N ==> A */ 32 #define CRYPTO_CMD_AARN 0x7 33 34 #define STARFIVE_RSA_MAX_KEYSZ 256 35 #define STARFIVE_RSA_RESET 0x2 36 37 static inline int starfive_pka_wait_done(struct starfive_cryp_ctx *ctx) 38 { 39 struct starfive_cryp_dev *cryp = ctx->cryp; 40 u32 status; 41 42 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_PKA_CASR_OFFSET, status, 43 status & STARFIVE_PKA_DONE, 10, 100000); 44 } 45 46 static void starfive_rsa_free_key(struct starfive_rsa_key *key) 47 { 48 if (!key->key_sz) 49 return; 50 51 kfree_sensitive(key->d); 52 kfree_sensitive(key->e); 53 kfree_sensitive(key->n); 54 memset(key, 0, sizeof(*key)); 55 } 56 57 static unsigned int starfive_rsa_get_nbit(u8 *pa, u32 snum, int key_sz) 58 { 59 u32 i; 60 u8 value; 61 62 i = snum >> 3; 63 64 value = pa[key_sz - i - 1]; 65 value >>= snum & 0x7; 66 value &= 0x1; 67 68 return value; 69 } 70 71 static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, 72 u32 *out, u32 *in, u8 mont, 73 u32 *mod, int bit_len) 74 { 75 struct starfive_cryp_dev *cryp = ctx->cryp; 76 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 77 int count = rctx->total / sizeof(u32) - 1; 78 int loop; 79 u32 temp; 80 u8 opsize; 81 82 opsize = (bit_len - 1) >> 5; 83 rctx->csr.pka.v = 0; 84 85 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 86 87 for (loop = 0; loop <= opsize; loop++) 88 writel(mod[opsize - loop], cryp->base + STARFIVE_PKA_CANR_OFFSET + loop * 4); 89 90 if (mont) { 91 rctx->csr.pka.v = 0; 92 rctx->csr.pka.cln_done = 1; 93 rctx->csr.pka.opsize = opsize; 94 rctx->csr.pka.exposize = opsize; 95 rctx->csr.pka.cmd = CRYPTO_CMD_PRE; 96 rctx->csr.pka.start = 1; 97 rctx->csr.pka.not_r2 = 1; 98 rctx->csr.pka.ie = 1; 99 100 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 101 102 if (starfive_pka_wait_done(ctx)) 103 return -ETIMEDOUT; 104 105 for (loop = 0; loop <= opsize; loop++) 106 writel(in[opsize - loop], cryp->base + STARFIVE_PKA_CAAR_OFFSET + loop * 4); 107 108 writel(0x1000000, cryp->base + STARFIVE_PKA_CAER_OFFSET); 109 110 for (loop = 1; loop <= opsize; loop++) 111 writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); 112 113 rctx->csr.pka.v = 0; 114 rctx->csr.pka.cln_done = 1; 115 rctx->csr.pka.opsize = opsize; 116 rctx->csr.pka.exposize = opsize; 117 rctx->csr.pka.cmd = CRYPTO_CMD_AERN; 118 rctx->csr.pka.start = 1; 119 rctx->csr.pka.ie = 1; 120 121 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 122 123 if (starfive_pka_wait_done(ctx)) 124 return -ETIMEDOUT; 125 } else { 126 rctx->csr.pka.v = 0; 127 rctx->csr.pka.cln_done = 1; 128 rctx->csr.pka.opsize = opsize; 129 rctx->csr.pka.exposize = opsize; 130 rctx->csr.pka.cmd = CRYPTO_CMD_PRE; 131 rctx->csr.pka.start = 1; 132 rctx->csr.pka.pre_expf = 1; 133 rctx->csr.pka.ie = 1; 134 135 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 136 137 if (starfive_pka_wait_done(ctx)) 138 return -ETIMEDOUT; 139 140 for (loop = 0; loop <= count; loop++) 141 writel(in[count - loop], cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); 142 143 /*pad with 0 up to opsize*/ 144 for (loop = count + 1; loop <= opsize; loop++) 145 writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); 146 147 rctx->csr.pka.v = 0; 148 rctx->csr.pka.cln_done = 1; 149 rctx->csr.pka.opsize = opsize; 150 rctx->csr.pka.exposize = opsize; 151 rctx->csr.pka.cmd = CRYPTO_CMD_ARN; 152 rctx->csr.pka.start = 1; 153 rctx->csr.pka.ie = 1; 154 155 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 156 157 if (starfive_pka_wait_done(ctx)) 158 return -ETIMEDOUT; 159 } 160 161 for (loop = 0; loop <= opsize; loop++) { 162 temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop); 163 out[opsize - loop] = temp; 164 } 165 166 return 0; 167 } 168 169 static int starfive_rsa_cpu_start(struct starfive_cryp_ctx *ctx, u32 *result, 170 u8 *de, u32 *n, int key_sz) 171 { 172 struct starfive_cryp_dev *cryp = ctx->cryp; 173 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 174 struct starfive_rsa_key *key = &ctx->rsa_key; 175 u32 temp; 176 int ret = 0; 177 int opsize, mlen, loop; 178 unsigned int *mta; 179 180 opsize = (key_sz - 1) >> 2; 181 182 mta = kmalloc(key_sz, GFP_KERNEL); 183 if (!mta) 184 return -ENOMEM; 185 186 ret = starfive_rsa_montgomery_form(ctx, mta, (u32 *)rctx->rsa_data, 187 0, n, key_sz << 3); 188 if (ret) { 189 dev_err_probe(cryp->dev, ret, "Conversion to Montgomery failed"); 190 goto rsa_err; 191 } 192 193 for (loop = 0; loop <= opsize; loop++) 194 writel(mta[opsize - loop], 195 cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); 196 197 for (loop = key->bitlen - 1; loop > 0; loop--) { 198 mlen = starfive_rsa_get_nbit(de, loop - 1, key_sz); 199 200 rctx->csr.pka.v = 0; 201 rctx->csr.pka.cln_done = 1; 202 rctx->csr.pka.opsize = opsize; 203 rctx->csr.pka.exposize = opsize; 204 rctx->csr.pka.cmd = CRYPTO_CMD_AARN; 205 rctx->csr.pka.start = 1; 206 rctx->csr.pka.ie = 1; 207 208 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 209 210 ret = -ETIMEDOUT; 211 if (starfive_pka_wait_done(ctx)) 212 goto rsa_err; 213 214 if (mlen) { 215 rctx->csr.pka.v = 0; 216 rctx->csr.pka.cln_done = 1; 217 rctx->csr.pka.opsize = opsize; 218 rctx->csr.pka.exposize = opsize; 219 rctx->csr.pka.cmd = CRYPTO_CMD_AERN; 220 rctx->csr.pka.start = 1; 221 rctx->csr.pka.ie = 1; 222 223 writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); 224 225 if (starfive_pka_wait_done(ctx)) 226 goto rsa_err; 227 } 228 } 229 230 for (loop = 0; loop <= opsize; loop++) { 231 temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop); 232 result[opsize - loop] = temp; 233 } 234 235 ret = starfive_rsa_montgomery_form(ctx, result, result, 1, n, key_sz << 3); 236 if (ret) 237 dev_err_probe(cryp->dev, ret, "Conversion from Montgomery failed"); 238 rsa_err: 239 kfree(mta); 240 return ret; 241 } 242 243 static int starfive_rsa_start(struct starfive_cryp_ctx *ctx, u8 *result, 244 u8 *de, u8 *n, int key_sz) 245 { 246 return starfive_rsa_cpu_start(ctx, (u32 *)result, de, (u32 *)n, key_sz); 247 } 248 249 static int starfive_rsa_enc_core(struct starfive_cryp_ctx *ctx, int enc) 250 { 251 struct starfive_cryp_dev *cryp = ctx->cryp; 252 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 253 struct starfive_rsa_key *key = &ctx->rsa_key; 254 int ret = 0; 255 256 writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); 257 258 rctx->total = sg_copy_to_buffer(rctx->in_sg, rctx->nents, 259 rctx->rsa_data, rctx->total); 260 261 if (enc) { 262 key->bitlen = key->e_bitlen; 263 ret = starfive_rsa_start(ctx, rctx->rsa_data, key->e, 264 key->n, key->key_sz); 265 } else { 266 key->bitlen = key->d_bitlen; 267 ret = starfive_rsa_start(ctx, rctx->rsa_data, key->d, 268 key->n, key->key_sz); 269 } 270 271 if (ret) 272 goto err_rsa_crypt; 273 274 sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), 275 rctx->rsa_data, key->key_sz, 0, 0); 276 277 err_rsa_crypt: 278 writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); 279 kfree(rctx->rsa_data); 280 return ret; 281 } 282 283 static int starfive_rsa_enc(struct akcipher_request *req) 284 { 285 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 286 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 287 struct starfive_cryp_dev *cryp = ctx->cryp; 288 struct starfive_rsa_key *key = &ctx->rsa_key; 289 struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req); 290 int ret; 291 292 if (!key->key_sz) { 293 akcipher_request_set_tfm(req, ctx->akcipher_fbk); 294 ret = crypto_akcipher_encrypt(req); 295 akcipher_request_set_tfm(req, tfm); 296 return ret; 297 } 298 299 if (unlikely(!key->n || !key->e)) 300 return -EINVAL; 301 302 if (req->dst_len < key->key_sz) 303 return dev_err_probe(cryp->dev, -EOVERFLOW, 304 "Output buffer length less than parameter n\n"); 305 306 rctx->in_sg = req->src; 307 rctx->out_sg = req->dst; 308 rctx->total = req->src_len; 309 rctx->nents = sg_nents(rctx->in_sg); 310 ctx->rctx = rctx; 311 312 return starfive_rsa_enc_core(ctx, 1); 313 } 314 315 static int starfive_rsa_dec(struct akcipher_request *req) 316 { 317 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 318 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 319 struct starfive_cryp_dev *cryp = ctx->cryp; 320 struct starfive_rsa_key *key = &ctx->rsa_key; 321 struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req); 322 int ret; 323 324 if (!key->key_sz) { 325 akcipher_request_set_tfm(req, ctx->akcipher_fbk); 326 ret = crypto_akcipher_decrypt(req); 327 akcipher_request_set_tfm(req, tfm); 328 return ret; 329 } 330 331 if (unlikely(!key->n || !key->d)) 332 return -EINVAL; 333 334 if (req->dst_len < key->key_sz) 335 return dev_err_probe(cryp->dev, -EOVERFLOW, 336 "Output buffer length less than parameter n\n"); 337 338 rctx->in_sg = req->src; 339 rctx->out_sg = req->dst; 340 ctx->rctx = rctx; 341 rctx->total = req->src_len; 342 343 return starfive_rsa_enc_core(ctx, 0); 344 } 345 346 static int starfive_rsa_set_n(struct starfive_rsa_key *rsa_key, 347 const char *value, size_t vlen) 348 { 349 const char *ptr = value; 350 unsigned int bitslen; 351 int ret; 352 353 while (!*ptr && vlen) { 354 ptr++; 355 vlen--; 356 } 357 rsa_key->key_sz = vlen; 358 bitslen = rsa_key->key_sz << 3; 359 360 /* check valid key size */ 361 if (bitslen & 0x1f) 362 return -EINVAL; 363 364 ret = -ENOMEM; 365 rsa_key->n = kmemdup(ptr, rsa_key->key_sz, GFP_KERNEL); 366 if (!rsa_key->n) 367 goto err; 368 369 return 0; 370 err: 371 rsa_key->key_sz = 0; 372 rsa_key->n = NULL; 373 starfive_rsa_free_key(rsa_key); 374 return ret; 375 } 376 377 static int starfive_rsa_set_e(struct starfive_rsa_key *rsa_key, 378 const char *value, size_t vlen) 379 { 380 const char *ptr = value; 381 unsigned char pt; 382 int loop; 383 384 while (!*ptr && vlen) { 385 ptr++; 386 vlen--; 387 } 388 pt = *ptr; 389 390 if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) { 391 rsa_key->e = NULL; 392 return -EINVAL; 393 } 394 395 rsa_key->e = kzalloc(rsa_key->key_sz, GFP_KERNEL); 396 if (!rsa_key->e) 397 return -ENOMEM; 398 399 for (loop = 8; loop > 0; loop--) { 400 if (pt >> (loop - 1)) 401 break; 402 } 403 404 rsa_key->e_bitlen = (vlen - 1) * 8 + loop; 405 406 memcpy(rsa_key->e + (rsa_key->key_sz - vlen), ptr, vlen); 407 408 return 0; 409 } 410 411 static int starfive_rsa_set_d(struct starfive_rsa_key *rsa_key, 412 const char *value, size_t vlen) 413 { 414 const char *ptr = value; 415 unsigned char pt; 416 int loop; 417 int ret; 418 419 while (!*ptr && vlen) { 420 ptr++; 421 vlen--; 422 } 423 pt = *ptr; 424 425 ret = -EINVAL; 426 if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) 427 goto err; 428 429 ret = -ENOMEM; 430 rsa_key->d = kzalloc(rsa_key->key_sz, GFP_KERNEL); 431 if (!rsa_key->d) 432 goto err; 433 434 for (loop = 8; loop > 0; loop--) { 435 if (pt >> (loop - 1)) 436 break; 437 } 438 439 rsa_key->d_bitlen = (vlen - 1) * 8 + loop; 440 441 memcpy(rsa_key->d + (rsa_key->key_sz - vlen), ptr, vlen); 442 443 return 0; 444 err: 445 rsa_key->d = NULL; 446 return ret; 447 } 448 449 static int starfive_rsa_setkey(struct crypto_akcipher *tfm, const void *key, 450 unsigned int keylen, bool private) 451 { 452 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 453 struct rsa_key raw_key = {NULL}; 454 struct starfive_rsa_key *rsa_key = &ctx->rsa_key; 455 int ret; 456 457 if (private) 458 ret = rsa_parse_priv_key(&raw_key, key, keylen); 459 else 460 ret = rsa_parse_pub_key(&raw_key, key, keylen); 461 if (ret < 0) 462 goto err; 463 464 starfive_rsa_free_key(rsa_key); 465 466 /* Use fallback for mod > 256 + 1 byte prefix */ 467 if (raw_key.n_sz > STARFIVE_RSA_MAX_KEYSZ + 1) 468 return 0; 469 470 ret = starfive_rsa_set_n(rsa_key, raw_key.n, raw_key.n_sz); 471 if (ret) 472 return ret; 473 474 ret = starfive_rsa_set_e(rsa_key, raw_key.e, raw_key.e_sz); 475 if (ret) 476 goto err; 477 478 if (private) { 479 ret = starfive_rsa_set_d(rsa_key, raw_key.d, raw_key.d_sz); 480 if (ret) 481 goto err; 482 } 483 484 if (!rsa_key->n || !rsa_key->e) { 485 ret = -EINVAL; 486 goto err; 487 } 488 489 if (private && !rsa_key->d) { 490 ret = -EINVAL; 491 goto err; 492 } 493 494 return 0; 495 err: 496 starfive_rsa_free_key(rsa_key); 497 return ret; 498 } 499 500 static int starfive_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, 501 unsigned int keylen) 502 { 503 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 504 int ret; 505 506 ret = crypto_akcipher_set_pub_key(ctx->akcipher_fbk, key, keylen); 507 if (ret) 508 return ret; 509 510 return starfive_rsa_setkey(tfm, key, keylen, false); 511 } 512 513 static int starfive_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, 514 unsigned int keylen) 515 { 516 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 517 int ret; 518 519 ret = crypto_akcipher_set_priv_key(ctx->akcipher_fbk, key, keylen); 520 if (ret) 521 return ret; 522 523 return starfive_rsa_setkey(tfm, key, keylen, true); 524 } 525 526 static unsigned int starfive_rsa_max_size(struct crypto_akcipher *tfm) 527 { 528 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 529 530 if (ctx->rsa_key.key_sz) 531 return ctx->rsa_key.key_sz; 532 533 return crypto_akcipher_maxsize(ctx->akcipher_fbk); 534 } 535 536 static int starfive_rsa_init_tfm(struct crypto_akcipher *tfm) 537 { 538 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 539 540 ctx->akcipher_fbk = crypto_alloc_akcipher("rsa-generic", 0, 0); 541 if (IS_ERR(ctx->akcipher_fbk)) 542 return PTR_ERR(ctx->akcipher_fbk); 543 544 ctx->cryp = starfive_cryp_find_dev(ctx); 545 if (!ctx->cryp) { 546 crypto_free_akcipher(ctx->akcipher_fbk); 547 return -ENODEV; 548 } 549 550 akcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + 551 sizeof(struct crypto_akcipher) + 32); 552 553 return 0; 554 } 555 556 static void starfive_rsa_exit_tfm(struct crypto_akcipher *tfm) 557 { 558 struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); 559 struct starfive_rsa_key *key = (struct starfive_rsa_key *)&ctx->rsa_key; 560 561 crypto_free_akcipher(ctx->akcipher_fbk); 562 starfive_rsa_free_key(key); 563 } 564 565 static struct akcipher_alg starfive_rsa = { 566 .encrypt = starfive_rsa_enc, 567 .decrypt = starfive_rsa_dec, 568 .sign = starfive_rsa_dec, 569 .verify = starfive_rsa_enc, 570 .set_pub_key = starfive_rsa_set_pub_key, 571 .set_priv_key = starfive_rsa_set_priv_key, 572 .max_size = starfive_rsa_max_size, 573 .init = starfive_rsa_init_tfm, 574 .exit = starfive_rsa_exit_tfm, 575 .base = { 576 .cra_name = "rsa", 577 .cra_driver_name = "starfive-rsa", 578 .cra_flags = CRYPTO_ALG_TYPE_AKCIPHER | 579 CRYPTO_ALG_NEED_FALLBACK, 580 .cra_priority = 3000, 581 .cra_module = THIS_MODULE, 582 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 583 }, 584 }; 585 586 int starfive_rsa_register_algs(void) 587 { 588 return crypto_register_akcipher(&starfive_rsa); 589 } 590 591 void starfive_rsa_unregister_algs(void) 592 { 593 crypto_unregister_akcipher(&starfive_rsa); 594 } 595