1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Cryptographic API for algorithms (i.e., low-level API). 4 * 5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 6 */ 7 8 #include <crypto/algapi.h> 9 #include <linux/err.h> 10 #include <linux/errno.h> 11 #include <linux/fips.h> 12 #include <linux/init.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/module.h> 16 #include <linux/rtnetlink.h> 17 #include <linux/slab.h> 18 #include <linux/string.h> 19 20 #include "internal.h" 21 22 static LIST_HEAD(crypto_template_list); 23 24 static inline void crypto_check_module_sig(struct module *mod) 25 { 26 if (fips_enabled && mod && !module_sig_ok(mod)) 27 panic("Module %s signature verification failed in FIPS mode\n", 28 module_name(mod)); 29 } 30 31 static int crypto_check_alg(struct crypto_alg *alg) 32 { 33 crypto_check_module_sig(alg->cra_module); 34 35 if (!alg->cra_name[0] || !alg->cra_driver_name[0]) 36 return -EINVAL; 37 38 if (alg->cra_alignmask & (alg->cra_alignmask + 1)) 39 return -EINVAL; 40 41 /* General maximums for all algs. */ 42 if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK) 43 return -EINVAL; 44 45 if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE) 46 return -EINVAL; 47 48 /* Lower maximums for specific alg types. */ 49 if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == 50 CRYPTO_ALG_TYPE_CIPHER) { 51 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK) 52 return -EINVAL; 53 54 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE) 55 return -EINVAL; 56 } 57 58 if (alg->cra_priority < 0) 59 return -EINVAL; 60 61 refcount_set(&alg->cra_refcnt, 1); 62 63 return 0; 64 } 65 66 static void crypto_free_instance(struct crypto_instance *inst) 67 { 68 if (!inst->alg.cra_type->free) { 69 inst->tmpl->free(inst); 70 return; 71 } 72 73 inst->alg.cra_type->free(inst); 74 } 75 76 static void crypto_destroy_instance(struct crypto_alg *alg) 77 { 78 struct crypto_instance *inst = (void *)alg; 79 struct crypto_template *tmpl = inst->tmpl; 80 81 crypto_free_instance(inst); 82 crypto_tmpl_put(tmpl); 83 } 84 85 /* 86 * This function adds a spawn to the list secondary_spawns which 87 * will be used at the end of crypto_remove_spawns to unregister 88 * instances, unless the spawn happens to be one that is depended 89 * on by the new algorithm (nalg in crypto_remove_spawns). 90 * 91 * This function is also responsible for resurrecting any algorithms 92 * in the dependency chain of nalg by unsetting n->dead. 93 */ 94 static struct list_head *crypto_more_spawns(struct crypto_alg *alg, 95 struct list_head *stack, 96 struct list_head *top, 97 struct list_head *secondary_spawns) 98 { 99 struct crypto_spawn *spawn, *n; 100 101 spawn = list_first_entry_or_null(stack, struct crypto_spawn, list); 102 if (!spawn) 103 return NULL; 104 105 n = list_prev_entry(spawn, list); 106 list_move(&spawn->list, secondary_spawns); 107 108 if (list_is_last(&n->list, stack)) 109 return top; 110 111 n = list_next_entry(n, list); 112 if (!spawn->dead) 113 n->dead = false; 114 115 return &n->inst->alg.cra_users; 116 } 117 118 static void crypto_remove_instance(struct crypto_instance *inst, 119 struct list_head *list) 120 { 121 struct crypto_template *tmpl = inst->tmpl; 122 123 if (crypto_is_dead(&inst->alg)) 124 return; 125 126 inst->alg.cra_flags |= CRYPTO_ALG_DEAD; 127 if (hlist_unhashed(&inst->list)) 128 return; 129 130 if (!tmpl || !crypto_tmpl_get(tmpl)) 131 return; 132 133 list_move(&inst->alg.cra_list, list); 134 hlist_del(&inst->list); 135 inst->alg.cra_destroy = crypto_destroy_instance; 136 137 BUG_ON(!list_empty(&inst->alg.cra_users)); 138 } 139 140 /* 141 * Given an algorithm alg, remove all algorithms that depend on it 142 * through spawns. If nalg is not null, then exempt any algorithms 143 * that is depended on by nalg. This is useful when nalg itself 144 * depends on alg. 145 */ 146 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, 147 struct crypto_alg *nalg) 148 { 149 u32 new_type = (nalg ?: alg)->cra_flags; 150 struct crypto_spawn *spawn, *n; 151 LIST_HEAD(secondary_spawns); 152 struct list_head *spawns; 153 LIST_HEAD(stack); 154 LIST_HEAD(top); 155 156 spawns = &alg->cra_users; 157 list_for_each_entry_safe(spawn, n, spawns, list) { 158 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask) 159 continue; 160 161 list_move(&spawn->list, &top); 162 } 163 164 /* 165 * Perform a depth-first walk starting from alg through 166 * the cra_users tree. The list stack records the path 167 * from alg to the current spawn. 168 */ 169 spawns = ⊤ 170 do { 171 while (!list_empty(spawns)) { 172 struct crypto_instance *inst; 173 174 spawn = list_first_entry(spawns, struct crypto_spawn, 175 list); 176 inst = spawn->inst; 177 178 BUG_ON(&inst->alg == alg); 179 180 list_move(&spawn->list, &stack); 181 182 if (&inst->alg == nalg) 183 break; 184 185 spawn->dead = true; 186 spawns = &inst->alg.cra_users; 187 188 /* 189 * We may encounter an unregistered instance here, since 190 * an instance's spawns are set up prior to the instance 191 * being registered. An unregistered instance will have 192 * NULL ->cra_users.next, since ->cra_users isn't 193 * properly initialized until registration. But an 194 * unregistered instance cannot have any users, so treat 195 * it the same as ->cra_users being empty. 196 */ 197 if (spawns->next == NULL) 198 break; 199 } 200 } while ((spawns = crypto_more_spawns(alg, &stack, &top, 201 &secondary_spawns))); 202 203 /* 204 * Remove all instances that are marked as dead. Also 205 * complete the resurrection of the others by moving them 206 * back to the cra_users list. 207 */ 208 list_for_each_entry_safe(spawn, n, &secondary_spawns, list) { 209 if (!spawn->dead) 210 list_move(&spawn->list, &spawn->alg->cra_users); 211 else 212 crypto_remove_instance(spawn->inst, list); 213 } 214 } 215 EXPORT_SYMBOL_GPL(crypto_remove_spawns); 216 217 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) 218 { 219 struct crypto_alg *q; 220 struct crypto_larval *larval; 221 int ret = -EAGAIN; 222 223 if (crypto_is_dead(alg)) 224 goto err; 225 226 INIT_LIST_HEAD(&alg->cra_users); 227 228 /* No cheating! */ 229 alg->cra_flags &= ~CRYPTO_ALG_TESTED; 230 231 ret = -EEXIST; 232 233 list_for_each_entry(q, &crypto_alg_list, cra_list) { 234 if (q == alg) 235 goto err; 236 237 if (crypto_is_moribund(q)) 238 continue; 239 240 if (crypto_is_larval(q)) { 241 if (!strcmp(alg->cra_driver_name, q->cra_driver_name)) 242 goto err; 243 continue; 244 } 245 246 if (!strcmp(q->cra_driver_name, alg->cra_name) || 247 !strcmp(q->cra_name, alg->cra_driver_name)) 248 goto err; 249 } 250 251 larval = crypto_larval_alloc(alg->cra_name, 252 alg->cra_flags | CRYPTO_ALG_TESTED, 0); 253 if (IS_ERR(larval)) 254 goto out; 255 256 ret = -ENOENT; 257 larval->adult = crypto_mod_get(alg); 258 if (!larval->adult) 259 goto free_larval; 260 261 refcount_set(&larval->alg.cra_refcnt, 1); 262 memcpy(larval->alg.cra_driver_name, alg->cra_driver_name, 263 CRYPTO_MAX_ALG_NAME); 264 larval->alg.cra_priority = alg->cra_priority; 265 266 list_add(&alg->cra_list, &crypto_alg_list); 267 list_add(&larval->alg.cra_list, &crypto_alg_list); 268 269 crypto_stats_init(alg); 270 271 out: 272 return larval; 273 274 free_larval: 275 kfree(larval); 276 err: 277 larval = ERR_PTR(ret); 278 goto out; 279 } 280 281 void crypto_alg_tested(const char *name, int err) 282 { 283 struct crypto_larval *test; 284 struct crypto_alg *alg; 285 struct crypto_alg *q; 286 LIST_HEAD(list); 287 bool best; 288 289 down_write(&crypto_alg_sem); 290 list_for_each_entry(q, &crypto_alg_list, cra_list) { 291 if (crypto_is_moribund(q) || !crypto_is_larval(q)) 292 continue; 293 294 test = (struct crypto_larval *)q; 295 296 if (!strcmp(q->cra_driver_name, name)) 297 goto found; 298 } 299 300 pr_err("alg: Unexpected test result for %s: %d\n", name, err); 301 goto unlock; 302 303 found: 304 q->cra_flags |= CRYPTO_ALG_DEAD; 305 alg = test->adult; 306 if (err || list_empty(&alg->cra_list)) 307 goto complete; 308 309 alg->cra_flags |= CRYPTO_ALG_TESTED; 310 311 /* Only satisfy larval waiters if we are the best. */ 312 best = true; 313 list_for_each_entry(q, &crypto_alg_list, cra_list) { 314 if (crypto_is_moribund(q) || !crypto_is_larval(q)) 315 continue; 316 317 if (strcmp(alg->cra_name, q->cra_name)) 318 continue; 319 320 if (q->cra_priority > alg->cra_priority) { 321 best = false; 322 break; 323 } 324 } 325 326 list_for_each_entry(q, &crypto_alg_list, cra_list) { 327 if (q == alg) 328 continue; 329 330 if (crypto_is_moribund(q)) 331 continue; 332 333 if (crypto_is_larval(q)) { 334 struct crypto_larval *larval = (void *)q; 335 336 /* 337 * Check to see if either our generic name or 338 * specific name can satisfy the name requested 339 * by the larval entry q. 340 */ 341 if (strcmp(alg->cra_name, q->cra_name) && 342 strcmp(alg->cra_driver_name, q->cra_name)) 343 continue; 344 345 if (larval->adult) 346 continue; 347 if ((q->cra_flags ^ alg->cra_flags) & larval->mask) 348 continue; 349 350 if (best && crypto_mod_get(alg)) 351 larval->adult = alg; 352 else 353 larval->adult = ERR_PTR(-EAGAIN); 354 355 continue; 356 } 357 358 if (strcmp(alg->cra_name, q->cra_name)) 359 continue; 360 361 if (strcmp(alg->cra_driver_name, q->cra_driver_name) && 362 q->cra_priority > alg->cra_priority) 363 continue; 364 365 crypto_remove_spawns(q, &list, alg); 366 } 367 368 complete: 369 complete_all(&test->completion); 370 371 unlock: 372 up_write(&crypto_alg_sem); 373 374 crypto_remove_final(&list); 375 } 376 EXPORT_SYMBOL_GPL(crypto_alg_tested); 377 378 void crypto_remove_final(struct list_head *list) 379 { 380 struct crypto_alg *alg; 381 struct crypto_alg *n; 382 383 list_for_each_entry_safe(alg, n, list, cra_list) { 384 list_del_init(&alg->cra_list); 385 crypto_alg_put(alg); 386 } 387 } 388 EXPORT_SYMBOL_GPL(crypto_remove_final); 389 390 static void crypto_wait_for_test(struct crypto_larval *larval) 391 { 392 int err; 393 394 err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult); 395 if (err != NOTIFY_STOP) { 396 if (WARN_ON(err != NOTIFY_DONE)) 397 goto out; 398 crypto_alg_tested(larval->alg.cra_driver_name, 0); 399 } 400 401 err = wait_for_completion_killable(&larval->completion); 402 WARN_ON(err); 403 if (!err) 404 crypto_probing_notify(CRYPTO_MSG_ALG_LOADED, larval); 405 406 out: 407 crypto_larval_kill(&larval->alg); 408 } 409 410 int crypto_register_alg(struct crypto_alg *alg) 411 { 412 struct crypto_larval *larval; 413 int err; 414 415 alg->cra_flags &= ~CRYPTO_ALG_DEAD; 416 err = crypto_check_alg(alg); 417 if (err) 418 return err; 419 420 down_write(&crypto_alg_sem); 421 larval = __crypto_register_alg(alg); 422 up_write(&crypto_alg_sem); 423 424 if (IS_ERR(larval)) 425 return PTR_ERR(larval); 426 427 crypto_wait_for_test(larval); 428 return 0; 429 } 430 EXPORT_SYMBOL_GPL(crypto_register_alg); 431 432 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) 433 { 434 if (unlikely(list_empty(&alg->cra_list))) 435 return -ENOENT; 436 437 alg->cra_flags |= CRYPTO_ALG_DEAD; 438 439 list_del_init(&alg->cra_list); 440 crypto_remove_spawns(alg, list, NULL); 441 442 return 0; 443 } 444 445 int crypto_unregister_alg(struct crypto_alg *alg) 446 { 447 int ret; 448 LIST_HEAD(list); 449 450 down_write(&crypto_alg_sem); 451 ret = crypto_remove_alg(alg, &list); 452 up_write(&crypto_alg_sem); 453 454 if (ret) 455 return ret; 456 457 BUG_ON(refcount_read(&alg->cra_refcnt) != 1); 458 if (alg->cra_destroy) 459 alg->cra_destroy(alg); 460 461 crypto_remove_final(&list); 462 return 0; 463 } 464 EXPORT_SYMBOL_GPL(crypto_unregister_alg); 465 466 int crypto_register_algs(struct crypto_alg *algs, int count) 467 { 468 int i, ret; 469 470 for (i = 0; i < count; i++) { 471 ret = crypto_register_alg(&algs[i]); 472 if (ret) 473 goto err; 474 } 475 476 return 0; 477 478 err: 479 for (--i; i >= 0; --i) 480 crypto_unregister_alg(&algs[i]); 481 482 return ret; 483 } 484 EXPORT_SYMBOL_GPL(crypto_register_algs); 485 486 int crypto_unregister_algs(struct crypto_alg *algs, int count) 487 { 488 int i, ret; 489 490 for (i = 0; i < count; i++) { 491 ret = crypto_unregister_alg(&algs[i]); 492 if (ret) 493 pr_err("Failed to unregister %s %s: %d\n", 494 algs[i].cra_driver_name, algs[i].cra_name, ret); 495 } 496 497 return 0; 498 } 499 EXPORT_SYMBOL_GPL(crypto_unregister_algs); 500 501 int crypto_register_template(struct crypto_template *tmpl) 502 { 503 struct crypto_template *q; 504 int err = -EEXIST; 505 506 down_write(&crypto_alg_sem); 507 508 crypto_check_module_sig(tmpl->module); 509 510 list_for_each_entry(q, &crypto_template_list, list) { 511 if (q == tmpl) 512 goto out; 513 } 514 515 list_add(&tmpl->list, &crypto_template_list); 516 err = 0; 517 out: 518 up_write(&crypto_alg_sem); 519 return err; 520 } 521 EXPORT_SYMBOL_GPL(crypto_register_template); 522 523 int crypto_register_templates(struct crypto_template *tmpls, int count) 524 { 525 int i, err; 526 527 for (i = 0; i < count; i++) { 528 err = crypto_register_template(&tmpls[i]); 529 if (err) 530 goto out; 531 } 532 return 0; 533 534 out: 535 for (--i; i >= 0; --i) 536 crypto_unregister_template(&tmpls[i]); 537 return err; 538 } 539 EXPORT_SYMBOL_GPL(crypto_register_templates); 540 541 void crypto_unregister_template(struct crypto_template *tmpl) 542 { 543 struct crypto_instance *inst; 544 struct hlist_node *n; 545 struct hlist_head *list; 546 LIST_HEAD(users); 547 548 down_write(&crypto_alg_sem); 549 550 BUG_ON(list_empty(&tmpl->list)); 551 list_del_init(&tmpl->list); 552 553 list = &tmpl->instances; 554 hlist_for_each_entry(inst, list, list) { 555 int err = crypto_remove_alg(&inst->alg, &users); 556 557 BUG_ON(err); 558 } 559 560 up_write(&crypto_alg_sem); 561 562 hlist_for_each_entry_safe(inst, n, list, list) { 563 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1); 564 crypto_free_instance(inst); 565 } 566 crypto_remove_final(&users); 567 } 568 EXPORT_SYMBOL_GPL(crypto_unregister_template); 569 570 void crypto_unregister_templates(struct crypto_template *tmpls, int count) 571 { 572 int i; 573 574 for (i = count - 1; i >= 0; --i) 575 crypto_unregister_template(&tmpls[i]); 576 } 577 EXPORT_SYMBOL_GPL(crypto_unregister_templates); 578 579 static struct crypto_template *__crypto_lookup_template(const char *name) 580 { 581 struct crypto_template *q, *tmpl = NULL; 582 583 down_read(&crypto_alg_sem); 584 list_for_each_entry(q, &crypto_template_list, list) { 585 if (strcmp(q->name, name)) 586 continue; 587 if (unlikely(!crypto_tmpl_get(q))) 588 continue; 589 590 tmpl = q; 591 break; 592 } 593 up_read(&crypto_alg_sem); 594 595 return tmpl; 596 } 597 598 struct crypto_template *crypto_lookup_template(const char *name) 599 { 600 return try_then_request_module(__crypto_lookup_template(name), 601 "crypto-%s", name); 602 } 603 EXPORT_SYMBOL_GPL(crypto_lookup_template); 604 605 int crypto_register_instance(struct crypto_template *tmpl, 606 struct crypto_instance *inst) 607 { 608 struct crypto_larval *larval; 609 int err; 610 611 err = crypto_check_alg(&inst->alg); 612 if (err) 613 return err; 614 615 inst->alg.cra_module = tmpl->module; 616 inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE; 617 618 down_write(&crypto_alg_sem); 619 620 larval = __crypto_register_alg(&inst->alg); 621 if (IS_ERR(larval)) 622 goto unlock; 623 624 hlist_add_head(&inst->list, &tmpl->instances); 625 inst->tmpl = tmpl; 626 627 unlock: 628 up_write(&crypto_alg_sem); 629 630 err = PTR_ERR(larval); 631 if (IS_ERR(larval)) 632 goto err; 633 634 crypto_wait_for_test(larval); 635 err = 0; 636 637 err: 638 return err; 639 } 640 EXPORT_SYMBOL_GPL(crypto_register_instance); 641 642 int crypto_unregister_instance(struct crypto_instance *inst) 643 { 644 LIST_HEAD(list); 645 646 down_write(&crypto_alg_sem); 647 648 crypto_remove_spawns(&inst->alg, &list, NULL); 649 crypto_remove_instance(inst, &list); 650 651 up_write(&crypto_alg_sem); 652 653 crypto_remove_final(&list); 654 655 return 0; 656 } 657 EXPORT_SYMBOL_GPL(crypto_unregister_instance); 658 659 int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, 660 struct crypto_instance *inst, u32 mask) 661 { 662 int err = -EAGAIN; 663 664 if (WARN_ON_ONCE(inst == NULL)) 665 return -EINVAL; 666 667 spawn->inst = inst; 668 spawn->mask = mask; 669 670 down_write(&crypto_alg_sem); 671 if (!crypto_is_moribund(alg)) { 672 list_add(&spawn->list, &alg->cra_users); 673 spawn->alg = alg; 674 err = 0; 675 } 676 up_write(&crypto_alg_sem); 677 678 return err; 679 } 680 EXPORT_SYMBOL_GPL(crypto_init_spawn); 681 682 int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, 683 struct crypto_instance *inst, 684 const struct crypto_type *frontend) 685 { 686 int err = -EINVAL; 687 688 if ((alg->cra_flags ^ frontend->type) & frontend->maskset) 689 goto out; 690 691 spawn->frontend = frontend; 692 err = crypto_init_spawn(spawn, alg, inst, frontend->maskset); 693 694 out: 695 return err; 696 } 697 EXPORT_SYMBOL_GPL(crypto_init_spawn2); 698 699 int crypto_grab_spawn(struct crypto_spawn *spawn, const char *name, 700 u32 type, u32 mask) 701 { 702 struct crypto_alg *alg; 703 int err; 704 705 alg = crypto_find_alg(name, spawn->frontend, type, mask); 706 if (IS_ERR(alg)) 707 return PTR_ERR(alg); 708 709 err = crypto_init_spawn(spawn, alg, spawn->inst, mask); 710 crypto_mod_put(alg); 711 return err; 712 } 713 EXPORT_SYMBOL_GPL(crypto_grab_spawn); 714 715 void crypto_drop_spawn(struct crypto_spawn *spawn) 716 { 717 down_write(&crypto_alg_sem); 718 if (!spawn->dead) 719 list_del(&spawn->list); 720 up_write(&crypto_alg_sem); 721 } 722 EXPORT_SYMBOL_GPL(crypto_drop_spawn); 723 724 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn) 725 { 726 struct crypto_alg *alg; 727 728 down_read(&crypto_alg_sem); 729 alg = spawn->alg; 730 if (!spawn->dead && !crypto_mod_get(alg)) { 731 alg->cra_flags |= CRYPTO_ALG_DYING; 732 alg = NULL; 733 } 734 up_read(&crypto_alg_sem); 735 736 return alg ?: ERR_PTR(-EAGAIN); 737 } 738 739 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, 740 u32 mask) 741 { 742 struct crypto_alg *alg; 743 struct crypto_tfm *tfm; 744 745 alg = crypto_spawn_alg(spawn); 746 if (IS_ERR(alg)) 747 return ERR_CAST(alg); 748 749 tfm = ERR_PTR(-EINVAL); 750 if (unlikely((alg->cra_flags ^ type) & mask)) 751 goto out_put_alg; 752 753 tfm = __crypto_alloc_tfm(alg, type, mask); 754 if (IS_ERR(tfm)) 755 goto out_put_alg; 756 757 return tfm; 758 759 out_put_alg: 760 crypto_mod_put(alg); 761 return tfm; 762 } 763 EXPORT_SYMBOL_GPL(crypto_spawn_tfm); 764 765 void *crypto_spawn_tfm2(struct crypto_spawn *spawn) 766 { 767 struct crypto_alg *alg; 768 struct crypto_tfm *tfm; 769 770 alg = crypto_spawn_alg(spawn); 771 if (IS_ERR(alg)) 772 return ERR_CAST(alg); 773 774 tfm = crypto_create_tfm(alg, spawn->frontend); 775 if (IS_ERR(tfm)) 776 goto out_put_alg; 777 778 return tfm; 779 780 out_put_alg: 781 crypto_mod_put(alg); 782 return tfm; 783 } 784 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2); 785 786 int crypto_register_notifier(struct notifier_block *nb) 787 { 788 return blocking_notifier_chain_register(&crypto_chain, nb); 789 } 790 EXPORT_SYMBOL_GPL(crypto_register_notifier); 791 792 int crypto_unregister_notifier(struct notifier_block *nb) 793 { 794 return blocking_notifier_chain_unregister(&crypto_chain, nb); 795 } 796 EXPORT_SYMBOL_GPL(crypto_unregister_notifier); 797 798 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb) 799 { 800 struct rtattr *rta = tb[0]; 801 struct crypto_attr_type *algt; 802 803 if (!rta) 804 return ERR_PTR(-ENOENT); 805 if (RTA_PAYLOAD(rta) < sizeof(*algt)) 806 return ERR_PTR(-EINVAL); 807 if (rta->rta_type != CRYPTOA_TYPE) 808 return ERR_PTR(-EINVAL); 809 810 algt = RTA_DATA(rta); 811 812 return algt; 813 } 814 EXPORT_SYMBOL_GPL(crypto_get_attr_type); 815 816 int crypto_check_attr_type(struct rtattr **tb, u32 type) 817 { 818 struct crypto_attr_type *algt; 819 820 algt = crypto_get_attr_type(tb); 821 if (IS_ERR(algt)) 822 return PTR_ERR(algt); 823 824 if ((algt->type ^ type) & algt->mask) 825 return -EINVAL; 826 827 return 0; 828 } 829 EXPORT_SYMBOL_GPL(crypto_check_attr_type); 830 831 const char *crypto_attr_alg_name(struct rtattr *rta) 832 { 833 struct crypto_attr_alg *alga; 834 835 if (!rta) 836 return ERR_PTR(-ENOENT); 837 if (RTA_PAYLOAD(rta) < sizeof(*alga)) 838 return ERR_PTR(-EINVAL); 839 if (rta->rta_type != CRYPTOA_ALG) 840 return ERR_PTR(-EINVAL); 841 842 alga = RTA_DATA(rta); 843 alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0; 844 845 return alga->name; 846 } 847 EXPORT_SYMBOL_GPL(crypto_attr_alg_name); 848 849 struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, 850 const struct crypto_type *frontend, 851 u32 type, u32 mask) 852 { 853 const char *name; 854 855 name = crypto_attr_alg_name(rta); 856 if (IS_ERR(name)) 857 return ERR_CAST(name); 858 859 return crypto_find_alg(name, frontend, type, mask); 860 } 861 EXPORT_SYMBOL_GPL(crypto_attr_alg2); 862 863 int crypto_attr_u32(struct rtattr *rta, u32 *num) 864 { 865 struct crypto_attr_u32 *nu32; 866 867 if (!rta) 868 return -ENOENT; 869 if (RTA_PAYLOAD(rta) < sizeof(*nu32)) 870 return -EINVAL; 871 if (rta->rta_type != CRYPTOA_U32) 872 return -EINVAL; 873 874 nu32 = RTA_DATA(rta); 875 *num = nu32->num; 876 877 return 0; 878 } 879 EXPORT_SYMBOL_GPL(crypto_attr_u32); 880 881 int crypto_inst_setname(struct crypto_instance *inst, const char *name, 882 struct crypto_alg *alg) 883 { 884 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, 885 alg->cra_name) >= CRYPTO_MAX_ALG_NAME) 886 return -ENAMETOOLONG; 887 888 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", 889 name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 890 return -ENAMETOOLONG; 891 892 return 0; 893 } 894 EXPORT_SYMBOL_GPL(crypto_inst_setname); 895 896 void *crypto_alloc_instance(const char *name, struct crypto_alg *alg, 897 unsigned int head) 898 { 899 struct crypto_instance *inst; 900 char *p; 901 int err; 902 903 p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn), 904 GFP_KERNEL); 905 if (!p) 906 return ERR_PTR(-ENOMEM); 907 908 inst = (void *)(p + head); 909 910 err = crypto_inst_setname(inst, name, alg); 911 if (err) 912 goto err_free_inst; 913 914 return p; 915 916 err_free_inst: 917 kfree(p); 918 return ERR_PTR(err); 919 } 920 EXPORT_SYMBOL_GPL(crypto_alloc_instance); 921 922 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen) 923 { 924 INIT_LIST_HEAD(&queue->list); 925 queue->backlog = &queue->list; 926 queue->qlen = 0; 927 queue->max_qlen = max_qlen; 928 } 929 EXPORT_SYMBOL_GPL(crypto_init_queue); 930 931 int crypto_enqueue_request(struct crypto_queue *queue, 932 struct crypto_async_request *request) 933 { 934 int err = -EINPROGRESS; 935 936 if (unlikely(queue->qlen >= queue->max_qlen)) { 937 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 938 err = -ENOSPC; 939 goto out; 940 } 941 err = -EBUSY; 942 if (queue->backlog == &queue->list) 943 queue->backlog = &request->list; 944 } 945 946 queue->qlen++; 947 list_add_tail(&request->list, &queue->list); 948 949 out: 950 return err; 951 } 952 EXPORT_SYMBOL_GPL(crypto_enqueue_request); 953 954 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) 955 { 956 struct list_head *request; 957 958 if (unlikely(!queue->qlen)) 959 return NULL; 960 961 queue->qlen--; 962 963 if (queue->backlog != &queue->list) 964 queue->backlog = queue->backlog->next; 965 966 request = queue->list.next; 967 list_del(request); 968 969 return list_entry(request, struct crypto_async_request, list); 970 } 971 EXPORT_SYMBOL_GPL(crypto_dequeue_request); 972 973 static inline void crypto_inc_byte(u8 *a, unsigned int size) 974 { 975 u8 *b = (a + size); 976 u8 c; 977 978 for (; size; size--) { 979 c = *--b + 1; 980 *b = c; 981 if (c) 982 break; 983 } 984 } 985 986 void crypto_inc(u8 *a, unsigned int size) 987 { 988 __be32 *b = (__be32 *)(a + size); 989 u32 c; 990 991 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || 992 IS_ALIGNED((unsigned long)b, __alignof__(*b))) 993 for (; size >= 4; size -= 4) { 994 c = be32_to_cpu(*--b) + 1; 995 *b = cpu_to_be32(c); 996 if (likely(c)) 997 return; 998 } 999 1000 crypto_inc_byte(a, size); 1001 } 1002 EXPORT_SYMBOL_GPL(crypto_inc); 1003 1004 void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) 1005 { 1006 int relalign = 0; 1007 1008 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { 1009 int size = sizeof(unsigned long); 1010 int d = (((unsigned long)dst ^ (unsigned long)src1) | 1011 ((unsigned long)dst ^ (unsigned long)src2)) & 1012 (size - 1); 1013 1014 relalign = d ? 1 << __ffs(d) : size; 1015 1016 /* 1017 * If we care about alignment, process as many bytes as 1018 * needed to advance dst and src to values whose alignments 1019 * equal their relative alignment. This will allow us to 1020 * process the remainder of the input using optimal strides. 1021 */ 1022 while (((unsigned long)dst & (relalign - 1)) && len > 0) { 1023 *dst++ = *src1++ ^ *src2++; 1024 len--; 1025 } 1026 } 1027 1028 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { 1029 *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; 1030 dst += 8; 1031 src1 += 8; 1032 src2 += 8; 1033 len -= 8; 1034 } 1035 1036 while (len >= 4 && !(relalign & 3)) { 1037 *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; 1038 dst += 4; 1039 src1 += 4; 1040 src2 += 4; 1041 len -= 4; 1042 } 1043 1044 while (len >= 2 && !(relalign & 1)) { 1045 *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; 1046 dst += 2; 1047 src1 += 2; 1048 src2 += 2; 1049 len -= 2; 1050 } 1051 1052 while (len--) 1053 *dst++ = *src1++ ^ *src2++; 1054 } 1055 EXPORT_SYMBOL_GPL(__crypto_xor); 1056 1057 unsigned int crypto_alg_extsize(struct crypto_alg *alg) 1058 { 1059 return alg->cra_ctxsize + 1060 (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1)); 1061 } 1062 EXPORT_SYMBOL_GPL(crypto_alg_extsize); 1063 1064 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend, 1065 u32 type, u32 mask) 1066 { 1067 int ret = 0; 1068 struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask); 1069 1070 if (!IS_ERR(alg)) { 1071 crypto_mod_put(alg); 1072 ret = 1; 1073 } 1074 1075 return ret; 1076 } 1077 EXPORT_SYMBOL_GPL(crypto_type_has_alg); 1078 1079 #ifdef CONFIG_CRYPTO_STATS 1080 void crypto_stats_init(struct crypto_alg *alg) 1081 { 1082 memset(&alg->stats, 0, sizeof(alg->stats)); 1083 } 1084 EXPORT_SYMBOL_GPL(crypto_stats_init); 1085 1086 void crypto_stats_get(struct crypto_alg *alg) 1087 { 1088 crypto_alg_get(alg); 1089 } 1090 EXPORT_SYMBOL_GPL(crypto_stats_get); 1091 1092 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg, 1093 int ret) 1094 { 1095 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1096 atomic64_inc(&alg->stats.aead.err_cnt); 1097 } else { 1098 atomic64_inc(&alg->stats.aead.encrypt_cnt); 1099 atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen); 1100 } 1101 crypto_alg_put(alg); 1102 } 1103 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt); 1104 1105 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg, 1106 int ret) 1107 { 1108 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1109 atomic64_inc(&alg->stats.aead.err_cnt); 1110 } else { 1111 atomic64_inc(&alg->stats.aead.decrypt_cnt); 1112 atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen); 1113 } 1114 crypto_alg_put(alg); 1115 } 1116 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt); 1117 1118 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret, 1119 struct crypto_alg *alg) 1120 { 1121 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1122 atomic64_inc(&alg->stats.akcipher.err_cnt); 1123 } else { 1124 atomic64_inc(&alg->stats.akcipher.encrypt_cnt); 1125 atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen); 1126 } 1127 crypto_alg_put(alg); 1128 } 1129 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt); 1130 1131 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret, 1132 struct crypto_alg *alg) 1133 { 1134 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1135 atomic64_inc(&alg->stats.akcipher.err_cnt); 1136 } else { 1137 atomic64_inc(&alg->stats.akcipher.decrypt_cnt); 1138 atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen); 1139 } 1140 crypto_alg_put(alg); 1141 } 1142 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt); 1143 1144 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg) 1145 { 1146 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1147 atomic64_inc(&alg->stats.akcipher.err_cnt); 1148 else 1149 atomic64_inc(&alg->stats.akcipher.sign_cnt); 1150 crypto_alg_put(alg); 1151 } 1152 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign); 1153 1154 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg) 1155 { 1156 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1157 atomic64_inc(&alg->stats.akcipher.err_cnt); 1158 else 1159 atomic64_inc(&alg->stats.akcipher.verify_cnt); 1160 crypto_alg_put(alg); 1161 } 1162 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify); 1163 1164 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg) 1165 { 1166 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1167 atomic64_inc(&alg->stats.compress.err_cnt); 1168 } else { 1169 atomic64_inc(&alg->stats.compress.compress_cnt); 1170 atomic64_add(slen, &alg->stats.compress.compress_tlen); 1171 } 1172 crypto_alg_put(alg); 1173 } 1174 EXPORT_SYMBOL_GPL(crypto_stats_compress); 1175 1176 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg) 1177 { 1178 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1179 atomic64_inc(&alg->stats.compress.err_cnt); 1180 } else { 1181 atomic64_inc(&alg->stats.compress.decompress_cnt); 1182 atomic64_add(slen, &alg->stats.compress.decompress_tlen); 1183 } 1184 crypto_alg_put(alg); 1185 } 1186 EXPORT_SYMBOL_GPL(crypto_stats_decompress); 1187 1188 void crypto_stats_ahash_update(unsigned int nbytes, int ret, 1189 struct crypto_alg *alg) 1190 { 1191 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1192 atomic64_inc(&alg->stats.hash.err_cnt); 1193 else 1194 atomic64_add(nbytes, &alg->stats.hash.hash_tlen); 1195 crypto_alg_put(alg); 1196 } 1197 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update); 1198 1199 void crypto_stats_ahash_final(unsigned int nbytes, int ret, 1200 struct crypto_alg *alg) 1201 { 1202 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1203 atomic64_inc(&alg->stats.hash.err_cnt); 1204 } else { 1205 atomic64_inc(&alg->stats.hash.hash_cnt); 1206 atomic64_add(nbytes, &alg->stats.hash.hash_tlen); 1207 } 1208 crypto_alg_put(alg); 1209 } 1210 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final); 1211 1212 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret) 1213 { 1214 if (ret) 1215 atomic64_inc(&alg->stats.kpp.err_cnt); 1216 else 1217 atomic64_inc(&alg->stats.kpp.setsecret_cnt); 1218 crypto_alg_put(alg); 1219 } 1220 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret); 1221 1222 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret) 1223 { 1224 if (ret) 1225 atomic64_inc(&alg->stats.kpp.err_cnt); 1226 else 1227 atomic64_inc(&alg->stats.kpp.generate_public_key_cnt); 1228 crypto_alg_put(alg); 1229 } 1230 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key); 1231 1232 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret) 1233 { 1234 if (ret) 1235 atomic64_inc(&alg->stats.kpp.err_cnt); 1236 else 1237 atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt); 1238 crypto_alg_put(alg); 1239 } 1240 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret); 1241 1242 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret) 1243 { 1244 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1245 atomic64_inc(&alg->stats.rng.err_cnt); 1246 else 1247 atomic64_inc(&alg->stats.rng.seed_cnt); 1248 crypto_alg_put(alg); 1249 } 1250 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed); 1251 1252 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen, 1253 int ret) 1254 { 1255 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1256 atomic64_inc(&alg->stats.rng.err_cnt); 1257 } else { 1258 atomic64_inc(&alg->stats.rng.generate_cnt); 1259 atomic64_add(dlen, &alg->stats.rng.generate_tlen); 1260 } 1261 crypto_alg_put(alg); 1262 } 1263 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate); 1264 1265 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret, 1266 struct crypto_alg *alg) 1267 { 1268 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1269 atomic64_inc(&alg->stats.cipher.err_cnt); 1270 } else { 1271 atomic64_inc(&alg->stats.cipher.encrypt_cnt); 1272 atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen); 1273 } 1274 crypto_alg_put(alg); 1275 } 1276 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt); 1277 1278 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret, 1279 struct crypto_alg *alg) 1280 { 1281 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1282 atomic64_inc(&alg->stats.cipher.err_cnt); 1283 } else { 1284 atomic64_inc(&alg->stats.cipher.decrypt_cnt); 1285 atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen); 1286 } 1287 crypto_alg_put(alg); 1288 } 1289 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt); 1290 #endif 1291 1292 static int __init crypto_algapi_init(void) 1293 { 1294 crypto_init_proc(); 1295 return 0; 1296 } 1297 1298 static void __exit crypto_algapi_exit(void) 1299 { 1300 crypto_exit_proc(); 1301 } 1302 1303 module_init(crypto_algapi_init); 1304 module_exit(crypto_algapi_exit); 1305 1306 MODULE_LICENSE("GPL"); 1307 MODULE_DESCRIPTION("Cryptographic algorithms API"); 1308