1 /* 2 * Cryptographic API for algorithms (i.e., low-level API). 3 * 4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the Free 8 * Software Foundation; either version 2 of the License, or (at your option) 9 * any later version. 10 * 11 */ 12 13 #include <crypto/algapi.h> 14 #include <linux/err.h> 15 #include <linux/errno.h> 16 #include <linux/fips.h> 17 #include <linux/init.h> 18 #include <linux/kernel.h> 19 #include <linux/list.h> 20 #include <linux/module.h> 21 #include <linux/rtnetlink.h> 22 #include <linux/slab.h> 23 #include <linux/string.h> 24 25 #include "internal.h" 26 27 static LIST_HEAD(crypto_template_list); 28 29 static inline int crypto_set_driver_name(struct crypto_alg *alg) 30 { 31 static const char suffix[] = "-generic"; 32 char *driver_name = alg->cra_driver_name; 33 int len; 34 35 if (*driver_name) 36 return 0; 37 38 len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 39 if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME) 40 return -ENAMETOOLONG; 41 42 memcpy(driver_name + len, suffix, sizeof(suffix)); 43 return 0; 44 } 45 46 static inline void crypto_check_module_sig(struct module *mod) 47 { 48 if (fips_enabled && mod && !module_sig_ok(mod)) 49 panic("Module %s signature verification failed in FIPS mode\n", 50 module_name(mod)); 51 } 52 53 static int crypto_check_alg(struct crypto_alg *alg) 54 { 55 crypto_check_module_sig(alg->cra_module); 56 57 if (alg->cra_alignmask & (alg->cra_alignmask + 1)) 58 return -EINVAL; 59 60 /* General maximums for all algs. */ 61 if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK) 62 return -EINVAL; 63 64 if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE) 65 return -EINVAL; 66 67 /* Lower maximums for specific alg types. */ 68 if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == 69 CRYPTO_ALG_TYPE_CIPHER) { 70 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK) 71 return -EINVAL; 72 73 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE) 74 return -EINVAL; 75 } 76 77 if (alg->cra_priority < 0) 78 return -EINVAL; 79 80 refcount_set(&alg->cra_refcnt, 1); 81 82 return crypto_set_driver_name(alg); 83 } 84 85 static void crypto_free_instance(struct crypto_instance *inst) 86 { 87 if (!inst->alg.cra_type->free) { 88 inst->tmpl->free(inst); 89 return; 90 } 91 92 inst->alg.cra_type->free(inst); 93 } 94 95 static void crypto_destroy_instance(struct crypto_alg *alg) 96 { 97 struct crypto_instance *inst = (void *)alg; 98 struct crypto_template *tmpl = inst->tmpl; 99 100 crypto_free_instance(inst); 101 crypto_tmpl_put(tmpl); 102 } 103 104 static struct list_head *crypto_more_spawns(struct crypto_alg *alg, 105 struct list_head *stack, 106 struct list_head *top, 107 struct list_head *secondary_spawns) 108 { 109 struct crypto_spawn *spawn, *n; 110 111 spawn = list_first_entry_or_null(stack, struct crypto_spawn, list); 112 if (!spawn) 113 return NULL; 114 115 n = list_next_entry(spawn, list); 116 117 if (spawn->alg && &n->list != stack && !n->alg) 118 n->alg = (n->list.next == stack) ? alg : 119 &list_next_entry(n, list)->inst->alg; 120 121 list_move(&spawn->list, secondary_spawns); 122 123 return &n->list == stack ? top : &n->inst->alg.cra_users; 124 } 125 126 static void crypto_remove_instance(struct crypto_instance *inst, 127 struct list_head *list) 128 { 129 struct crypto_template *tmpl = inst->tmpl; 130 131 if (crypto_is_dead(&inst->alg)) 132 return; 133 134 inst->alg.cra_flags |= CRYPTO_ALG_DEAD; 135 if (hlist_unhashed(&inst->list)) 136 return; 137 138 if (!tmpl || !crypto_tmpl_get(tmpl)) 139 return; 140 141 list_move(&inst->alg.cra_list, list); 142 hlist_del(&inst->list); 143 inst->alg.cra_destroy = crypto_destroy_instance; 144 145 BUG_ON(!list_empty(&inst->alg.cra_users)); 146 } 147 148 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, 149 struct crypto_alg *nalg) 150 { 151 u32 new_type = (nalg ?: alg)->cra_flags; 152 struct crypto_spawn *spawn, *n; 153 LIST_HEAD(secondary_spawns); 154 struct list_head *spawns; 155 LIST_HEAD(stack); 156 LIST_HEAD(top); 157 158 spawns = &alg->cra_users; 159 list_for_each_entry_safe(spawn, n, spawns, list) { 160 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask) 161 continue; 162 163 list_move(&spawn->list, &top); 164 } 165 166 spawns = ⊤ 167 do { 168 while (!list_empty(spawns)) { 169 struct crypto_instance *inst; 170 171 spawn = list_first_entry(spawns, struct crypto_spawn, 172 list); 173 inst = spawn->inst; 174 175 BUG_ON(&inst->alg == alg); 176 177 list_move(&spawn->list, &stack); 178 179 if (&inst->alg == nalg) 180 break; 181 182 spawn->alg = NULL; 183 spawns = &inst->alg.cra_users; 184 185 /* 186 * We may encounter an unregistered instance here, since 187 * an instance's spawns are set up prior to the instance 188 * being registered. An unregistered instance will have 189 * NULL ->cra_users.next, since ->cra_users isn't 190 * properly initialized until registration. But an 191 * unregistered instance cannot have any users, so treat 192 * it the same as ->cra_users being empty. 193 */ 194 if (spawns->next == NULL) 195 break; 196 } 197 } while ((spawns = crypto_more_spawns(alg, &stack, &top, 198 &secondary_spawns))); 199 200 list_for_each_entry_safe(spawn, n, &secondary_spawns, list) { 201 if (spawn->alg) 202 list_move(&spawn->list, &spawn->alg->cra_users); 203 else 204 crypto_remove_instance(spawn->inst, list); 205 } 206 } 207 EXPORT_SYMBOL_GPL(crypto_remove_spawns); 208 209 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) 210 { 211 struct crypto_alg *q; 212 struct crypto_larval *larval; 213 int ret = -EAGAIN; 214 215 if (crypto_is_dead(alg)) 216 goto err; 217 218 INIT_LIST_HEAD(&alg->cra_users); 219 220 /* No cheating! */ 221 alg->cra_flags &= ~CRYPTO_ALG_TESTED; 222 223 ret = -EEXIST; 224 225 list_for_each_entry(q, &crypto_alg_list, cra_list) { 226 if (q == alg) 227 goto err; 228 229 if (crypto_is_moribund(q)) 230 continue; 231 232 if (crypto_is_larval(q)) { 233 if (!strcmp(alg->cra_driver_name, q->cra_driver_name)) 234 goto err; 235 continue; 236 } 237 238 if (!strcmp(q->cra_driver_name, alg->cra_name) || 239 !strcmp(q->cra_name, alg->cra_driver_name)) 240 goto err; 241 } 242 243 larval = crypto_larval_alloc(alg->cra_name, 244 alg->cra_flags | CRYPTO_ALG_TESTED, 0); 245 if (IS_ERR(larval)) 246 goto out; 247 248 ret = -ENOENT; 249 larval->adult = crypto_mod_get(alg); 250 if (!larval->adult) 251 goto free_larval; 252 253 refcount_set(&larval->alg.cra_refcnt, 1); 254 memcpy(larval->alg.cra_driver_name, alg->cra_driver_name, 255 CRYPTO_MAX_ALG_NAME); 256 larval->alg.cra_priority = alg->cra_priority; 257 258 list_add(&alg->cra_list, &crypto_alg_list); 259 list_add(&larval->alg.cra_list, &crypto_alg_list); 260 261 #ifdef CONFIG_CRYPTO_STATS 262 memset(&alg->stats, 0, sizeof(alg->stats)); 263 #endif 264 265 out: 266 return larval; 267 268 free_larval: 269 kfree(larval); 270 err: 271 larval = ERR_PTR(ret); 272 goto out; 273 } 274 275 void crypto_alg_tested(const char *name, int err) 276 { 277 struct crypto_larval *test; 278 struct crypto_alg *alg; 279 struct crypto_alg *q; 280 LIST_HEAD(list); 281 282 down_write(&crypto_alg_sem); 283 list_for_each_entry(q, &crypto_alg_list, cra_list) { 284 if (crypto_is_moribund(q) || !crypto_is_larval(q)) 285 continue; 286 287 test = (struct crypto_larval *)q; 288 289 if (!strcmp(q->cra_driver_name, name)) 290 goto found; 291 } 292 293 pr_err("alg: Unexpected test result for %s: %d\n", name, err); 294 goto unlock; 295 296 found: 297 q->cra_flags |= CRYPTO_ALG_DEAD; 298 alg = test->adult; 299 if (err || list_empty(&alg->cra_list)) 300 goto complete; 301 302 alg->cra_flags |= CRYPTO_ALG_TESTED; 303 304 list_for_each_entry(q, &crypto_alg_list, cra_list) { 305 if (q == alg) 306 continue; 307 308 if (crypto_is_moribund(q)) 309 continue; 310 311 if (crypto_is_larval(q)) { 312 struct crypto_larval *larval = (void *)q; 313 314 /* 315 * Check to see if either our generic name or 316 * specific name can satisfy the name requested 317 * by the larval entry q. 318 */ 319 if (strcmp(alg->cra_name, q->cra_name) && 320 strcmp(alg->cra_driver_name, q->cra_name)) 321 continue; 322 323 if (larval->adult) 324 continue; 325 if ((q->cra_flags ^ alg->cra_flags) & larval->mask) 326 continue; 327 if (!crypto_mod_get(alg)) 328 continue; 329 330 larval->adult = alg; 331 continue; 332 } 333 334 if (strcmp(alg->cra_name, q->cra_name)) 335 continue; 336 337 if (strcmp(alg->cra_driver_name, q->cra_driver_name) && 338 q->cra_priority > alg->cra_priority) 339 continue; 340 341 crypto_remove_spawns(q, &list, alg); 342 } 343 344 complete: 345 complete_all(&test->completion); 346 347 unlock: 348 up_write(&crypto_alg_sem); 349 350 crypto_remove_final(&list); 351 } 352 EXPORT_SYMBOL_GPL(crypto_alg_tested); 353 354 void crypto_remove_final(struct list_head *list) 355 { 356 struct crypto_alg *alg; 357 struct crypto_alg *n; 358 359 list_for_each_entry_safe(alg, n, list, cra_list) { 360 list_del_init(&alg->cra_list); 361 crypto_alg_put(alg); 362 } 363 } 364 EXPORT_SYMBOL_GPL(crypto_remove_final); 365 366 static void crypto_wait_for_test(struct crypto_larval *larval) 367 { 368 int err; 369 370 err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult); 371 if (err != NOTIFY_STOP) { 372 if (WARN_ON(err != NOTIFY_DONE)) 373 goto out; 374 crypto_alg_tested(larval->alg.cra_driver_name, 0); 375 } 376 377 err = wait_for_completion_killable(&larval->completion); 378 WARN_ON(err); 379 if (!err) 380 crypto_probing_notify(CRYPTO_MSG_ALG_LOADED, larval); 381 382 out: 383 crypto_larval_kill(&larval->alg); 384 } 385 386 int crypto_register_alg(struct crypto_alg *alg) 387 { 388 struct crypto_larval *larval; 389 int err; 390 391 alg->cra_flags &= ~CRYPTO_ALG_DEAD; 392 err = crypto_check_alg(alg); 393 if (err) 394 return err; 395 396 down_write(&crypto_alg_sem); 397 larval = __crypto_register_alg(alg); 398 up_write(&crypto_alg_sem); 399 400 if (IS_ERR(larval)) 401 return PTR_ERR(larval); 402 403 crypto_wait_for_test(larval); 404 return 0; 405 } 406 EXPORT_SYMBOL_GPL(crypto_register_alg); 407 408 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) 409 { 410 if (unlikely(list_empty(&alg->cra_list))) 411 return -ENOENT; 412 413 alg->cra_flags |= CRYPTO_ALG_DEAD; 414 415 list_del_init(&alg->cra_list); 416 crypto_remove_spawns(alg, list, NULL); 417 418 return 0; 419 } 420 421 int crypto_unregister_alg(struct crypto_alg *alg) 422 { 423 int ret; 424 LIST_HEAD(list); 425 426 down_write(&crypto_alg_sem); 427 ret = crypto_remove_alg(alg, &list); 428 up_write(&crypto_alg_sem); 429 430 if (ret) 431 return ret; 432 433 BUG_ON(refcount_read(&alg->cra_refcnt) != 1); 434 if (alg->cra_destroy) 435 alg->cra_destroy(alg); 436 437 crypto_remove_final(&list); 438 return 0; 439 } 440 EXPORT_SYMBOL_GPL(crypto_unregister_alg); 441 442 int crypto_register_algs(struct crypto_alg *algs, int count) 443 { 444 int i, ret; 445 446 for (i = 0; i < count; i++) { 447 ret = crypto_register_alg(&algs[i]); 448 if (ret) 449 goto err; 450 } 451 452 return 0; 453 454 err: 455 for (--i; i >= 0; --i) 456 crypto_unregister_alg(&algs[i]); 457 458 return ret; 459 } 460 EXPORT_SYMBOL_GPL(crypto_register_algs); 461 462 int crypto_unregister_algs(struct crypto_alg *algs, int count) 463 { 464 int i, ret; 465 466 for (i = 0; i < count; i++) { 467 ret = crypto_unregister_alg(&algs[i]); 468 if (ret) 469 pr_err("Failed to unregister %s %s: %d\n", 470 algs[i].cra_driver_name, algs[i].cra_name, ret); 471 } 472 473 return 0; 474 } 475 EXPORT_SYMBOL_GPL(crypto_unregister_algs); 476 477 int crypto_register_template(struct crypto_template *tmpl) 478 { 479 struct crypto_template *q; 480 int err = -EEXIST; 481 482 down_write(&crypto_alg_sem); 483 484 crypto_check_module_sig(tmpl->module); 485 486 list_for_each_entry(q, &crypto_template_list, list) { 487 if (q == tmpl) 488 goto out; 489 } 490 491 list_add(&tmpl->list, &crypto_template_list); 492 err = 0; 493 out: 494 up_write(&crypto_alg_sem); 495 return err; 496 } 497 EXPORT_SYMBOL_GPL(crypto_register_template); 498 499 void crypto_unregister_template(struct crypto_template *tmpl) 500 { 501 struct crypto_instance *inst; 502 struct hlist_node *n; 503 struct hlist_head *list; 504 LIST_HEAD(users); 505 506 down_write(&crypto_alg_sem); 507 508 BUG_ON(list_empty(&tmpl->list)); 509 list_del_init(&tmpl->list); 510 511 list = &tmpl->instances; 512 hlist_for_each_entry(inst, list, list) { 513 int err = crypto_remove_alg(&inst->alg, &users); 514 515 BUG_ON(err); 516 } 517 518 up_write(&crypto_alg_sem); 519 520 hlist_for_each_entry_safe(inst, n, list, list) { 521 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1); 522 crypto_free_instance(inst); 523 } 524 crypto_remove_final(&users); 525 } 526 EXPORT_SYMBOL_GPL(crypto_unregister_template); 527 528 static struct crypto_template *__crypto_lookup_template(const char *name) 529 { 530 struct crypto_template *q, *tmpl = NULL; 531 532 down_read(&crypto_alg_sem); 533 list_for_each_entry(q, &crypto_template_list, list) { 534 if (strcmp(q->name, name)) 535 continue; 536 if (unlikely(!crypto_tmpl_get(q))) 537 continue; 538 539 tmpl = q; 540 break; 541 } 542 up_read(&crypto_alg_sem); 543 544 return tmpl; 545 } 546 547 struct crypto_template *crypto_lookup_template(const char *name) 548 { 549 return try_then_request_module(__crypto_lookup_template(name), 550 "crypto-%s", name); 551 } 552 EXPORT_SYMBOL_GPL(crypto_lookup_template); 553 554 int crypto_register_instance(struct crypto_template *tmpl, 555 struct crypto_instance *inst) 556 { 557 struct crypto_larval *larval; 558 int err; 559 560 err = crypto_check_alg(&inst->alg); 561 if (err) 562 return err; 563 564 inst->alg.cra_module = tmpl->module; 565 inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE; 566 567 down_write(&crypto_alg_sem); 568 569 larval = __crypto_register_alg(&inst->alg); 570 if (IS_ERR(larval)) 571 goto unlock; 572 573 hlist_add_head(&inst->list, &tmpl->instances); 574 inst->tmpl = tmpl; 575 576 unlock: 577 up_write(&crypto_alg_sem); 578 579 err = PTR_ERR(larval); 580 if (IS_ERR(larval)) 581 goto err; 582 583 crypto_wait_for_test(larval); 584 err = 0; 585 586 err: 587 return err; 588 } 589 EXPORT_SYMBOL_GPL(crypto_register_instance); 590 591 int crypto_unregister_instance(struct crypto_instance *inst) 592 { 593 LIST_HEAD(list); 594 595 down_write(&crypto_alg_sem); 596 597 crypto_remove_spawns(&inst->alg, &list, NULL); 598 crypto_remove_instance(inst, &list); 599 600 up_write(&crypto_alg_sem); 601 602 crypto_remove_final(&list); 603 604 return 0; 605 } 606 EXPORT_SYMBOL_GPL(crypto_unregister_instance); 607 608 int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, 609 struct crypto_instance *inst, u32 mask) 610 { 611 int err = -EAGAIN; 612 613 spawn->inst = inst; 614 spawn->mask = mask; 615 616 down_write(&crypto_alg_sem); 617 if (!crypto_is_moribund(alg)) { 618 list_add(&spawn->list, &alg->cra_users); 619 spawn->alg = alg; 620 err = 0; 621 } 622 up_write(&crypto_alg_sem); 623 624 return err; 625 } 626 EXPORT_SYMBOL_GPL(crypto_init_spawn); 627 628 int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, 629 struct crypto_instance *inst, 630 const struct crypto_type *frontend) 631 { 632 int err = -EINVAL; 633 634 if ((alg->cra_flags ^ frontend->type) & frontend->maskset) 635 goto out; 636 637 spawn->frontend = frontend; 638 err = crypto_init_spawn(spawn, alg, inst, frontend->maskset); 639 640 out: 641 return err; 642 } 643 EXPORT_SYMBOL_GPL(crypto_init_spawn2); 644 645 int crypto_grab_spawn(struct crypto_spawn *spawn, const char *name, 646 u32 type, u32 mask) 647 { 648 struct crypto_alg *alg; 649 int err; 650 651 alg = crypto_find_alg(name, spawn->frontend, type, mask); 652 if (IS_ERR(alg)) 653 return PTR_ERR(alg); 654 655 err = crypto_init_spawn(spawn, alg, spawn->inst, mask); 656 crypto_mod_put(alg); 657 return err; 658 } 659 EXPORT_SYMBOL_GPL(crypto_grab_spawn); 660 661 void crypto_drop_spawn(struct crypto_spawn *spawn) 662 { 663 if (!spawn->alg) 664 return; 665 666 down_write(&crypto_alg_sem); 667 list_del(&spawn->list); 668 up_write(&crypto_alg_sem); 669 } 670 EXPORT_SYMBOL_GPL(crypto_drop_spawn); 671 672 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn) 673 { 674 struct crypto_alg *alg; 675 struct crypto_alg *alg2; 676 677 down_read(&crypto_alg_sem); 678 alg = spawn->alg; 679 alg2 = alg; 680 if (alg2) 681 alg2 = crypto_mod_get(alg2); 682 up_read(&crypto_alg_sem); 683 684 if (!alg2) { 685 if (alg) 686 crypto_shoot_alg(alg); 687 return ERR_PTR(-EAGAIN); 688 } 689 690 return alg; 691 } 692 693 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, 694 u32 mask) 695 { 696 struct crypto_alg *alg; 697 struct crypto_tfm *tfm; 698 699 alg = crypto_spawn_alg(spawn); 700 if (IS_ERR(alg)) 701 return ERR_CAST(alg); 702 703 tfm = ERR_PTR(-EINVAL); 704 if (unlikely((alg->cra_flags ^ type) & mask)) 705 goto out_put_alg; 706 707 tfm = __crypto_alloc_tfm(alg, type, mask); 708 if (IS_ERR(tfm)) 709 goto out_put_alg; 710 711 return tfm; 712 713 out_put_alg: 714 crypto_mod_put(alg); 715 return tfm; 716 } 717 EXPORT_SYMBOL_GPL(crypto_spawn_tfm); 718 719 void *crypto_spawn_tfm2(struct crypto_spawn *spawn) 720 { 721 struct crypto_alg *alg; 722 struct crypto_tfm *tfm; 723 724 alg = crypto_spawn_alg(spawn); 725 if (IS_ERR(alg)) 726 return ERR_CAST(alg); 727 728 tfm = crypto_create_tfm(alg, spawn->frontend); 729 if (IS_ERR(tfm)) 730 goto out_put_alg; 731 732 return tfm; 733 734 out_put_alg: 735 crypto_mod_put(alg); 736 return tfm; 737 } 738 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2); 739 740 int crypto_register_notifier(struct notifier_block *nb) 741 { 742 return blocking_notifier_chain_register(&crypto_chain, nb); 743 } 744 EXPORT_SYMBOL_GPL(crypto_register_notifier); 745 746 int crypto_unregister_notifier(struct notifier_block *nb) 747 { 748 return blocking_notifier_chain_unregister(&crypto_chain, nb); 749 } 750 EXPORT_SYMBOL_GPL(crypto_unregister_notifier); 751 752 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb) 753 { 754 struct rtattr *rta = tb[0]; 755 struct crypto_attr_type *algt; 756 757 if (!rta) 758 return ERR_PTR(-ENOENT); 759 if (RTA_PAYLOAD(rta) < sizeof(*algt)) 760 return ERR_PTR(-EINVAL); 761 if (rta->rta_type != CRYPTOA_TYPE) 762 return ERR_PTR(-EINVAL); 763 764 algt = RTA_DATA(rta); 765 766 return algt; 767 } 768 EXPORT_SYMBOL_GPL(crypto_get_attr_type); 769 770 int crypto_check_attr_type(struct rtattr **tb, u32 type) 771 { 772 struct crypto_attr_type *algt; 773 774 algt = crypto_get_attr_type(tb); 775 if (IS_ERR(algt)) 776 return PTR_ERR(algt); 777 778 if ((algt->type ^ type) & algt->mask) 779 return -EINVAL; 780 781 return 0; 782 } 783 EXPORT_SYMBOL_GPL(crypto_check_attr_type); 784 785 const char *crypto_attr_alg_name(struct rtattr *rta) 786 { 787 struct crypto_attr_alg *alga; 788 789 if (!rta) 790 return ERR_PTR(-ENOENT); 791 if (RTA_PAYLOAD(rta) < sizeof(*alga)) 792 return ERR_PTR(-EINVAL); 793 if (rta->rta_type != CRYPTOA_ALG) 794 return ERR_PTR(-EINVAL); 795 796 alga = RTA_DATA(rta); 797 alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0; 798 799 return alga->name; 800 } 801 EXPORT_SYMBOL_GPL(crypto_attr_alg_name); 802 803 struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, 804 const struct crypto_type *frontend, 805 u32 type, u32 mask) 806 { 807 const char *name; 808 809 name = crypto_attr_alg_name(rta); 810 if (IS_ERR(name)) 811 return ERR_CAST(name); 812 813 return crypto_find_alg(name, frontend, type, mask); 814 } 815 EXPORT_SYMBOL_GPL(crypto_attr_alg2); 816 817 int crypto_attr_u32(struct rtattr *rta, u32 *num) 818 { 819 struct crypto_attr_u32 *nu32; 820 821 if (!rta) 822 return -ENOENT; 823 if (RTA_PAYLOAD(rta) < sizeof(*nu32)) 824 return -EINVAL; 825 if (rta->rta_type != CRYPTOA_U32) 826 return -EINVAL; 827 828 nu32 = RTA_DATA(rta); 829 *num = nu32->num; 830 831 return 0; 832 } 833 EXPORT_SYMBOL_GPL(crypto_attr_u32); 834 835 int crypto_inst_setname(struct crypto_instance *inst, const char *name, 836 struct crypto_alg *alg) 837 { 838 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, 839 alg->cra_name) >= CRYPTO_MAX_ALG_NAME) 840 return -ENAMETOOLONG; 841 842 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", 843 name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 844 return -ENAMETOOLONG; 845 846 return 0; 847 } 848 EXPORT_SYMBOL_GPL(crypto_inst_setname); 849 850 void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, 851 unsigned int head) 852 { 853 struct crypto_instance *inst; 854 char *p; 855 int err; 856 857 p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn), 858 GFP_KERNEL); 859 if (!p) 860 return ERR_PTR(-ENOMEM); 861 862 inst = (void *)(p + head); 863 864 err = crypto_inst_setname(inst, name, alg); 865 if (err) 866 goto err_free_inst; 867 868 return p; 869 870 err_free_inst: 871 kfree(p); 872 return ERR_PTR(err); 873 } 874 EXPORT_SYMBOL_GPL(crypto_alloc_instance2); 875 876 struct crypto_instance *crypto_alloc_instance(const char *name, 877 struct crypto_alg *alg) 878 { 879 struct crypto_instance *inst; 880 struct crypto_spawn *spawn; 881 int err; 882 883 inst = crypto_alloc_instance2(name, alg, 0); 884 if (IS_ERR(inst)) 885 goto out; 886 887 spawn = crypto_instance_ctx(inst); 888 err = crypto_init_spawn(spawn, alg, inst, 889 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); 890 891 if (err) 892 goto err_free_inst; 893 894 return inst; 895 896 err_free_inst: 897 kfree(inst); 898 inst = ERR_PTR(err); 899 900 out: 901 return inst; 902 } 903 EXPORT_SYMBOL_GPL(crypto_alloc_instance); 904 905 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen) 906 { 907 INIT_LIST_HEAD(&queue->list); 908 queue->backlog = &queue->list; 909 queue->qlen = 0; 910 queue->max_qlen = max_qlen; 911 } 912 EXPORT_SYMBOL_GPL(crypto_init_queue); 913 914 int crypto_enqueue_request(struct crypto_queue *queue, 915 struct crypto_async_request *request) 916 { 917 int err = -EINPROGRESS; 918 919 if (unlikely(queue->qlen >= queue->max_qlen)) { 920 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 921 err = -ENOSPC; 922 goto out; 923 } 924 err = -EBUSY; 925 if (queue->backlog == &queue->list) 926 queue->backlog = &request->list; 927 } 928 929 queue->qlen++; 930 list_add_tail(&request->list, &queue->list); 931 932 out: 933 return err; 934 } 935 EXPORT_SYMBOL_GPL(crypto_enqueue_request); 936 937 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) 938 { 939 struct list_head *request; 940 941 if (unlikely(!queue->qlen)) 942 return NULL; 943 944 queue->qlen--; 945 946 if (queue->backlog != &queue->list) 947 queue->backlog = queue->backlog->next; 948 949 request = queue->list.next; 950 list_del(request); 951 952 return list_entry(request, struct crypto_async_request, list); 953 } 954 EXPORT_SYMBOL_GPL(crypto_dequeue_request); 955 956 int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm) 957 { 958 struct crypto_async_request *req; 959 960 list_for_each_entry(req, &queue->list, list) { 961 if (req->tfm == tfm) 962 return 1; 963 } 964 965 return 0; 966 } 967 EXPORT_SYMBOL_GPL(crypto_tfm_in_queue); 968 969 static inline void crypto_inc_byte(u8 *a, unsigned int size) 970 { 971 u8 *b = (a + size); 972 u8 c; 973 974 for (; size; size--) { 975 c = *--b + 1; 976 *b = c; 977 if (c) 978 break; 979 } 980 } 981 982 void crypto_inc(u8 *a, unsigned int size) 983 { 984 __be32 *b = (__be32 *)(a + size); 985 u32 c; 986 987 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || 988 IS_ALIGNED((unsigned long)b, __alignof__(*b))) 989 for (; size >= 4; size -= 4) { 990 c = be32_to_cpu(*--b) + 1; 991 *b = cpu_to_be32(c); 992 if (likely(c)) 993 return; 994 } 995 996 crypto_inc_byte(a, size); 997 } 998 EXPORT_SYMBOL_GPL(crypto_inc); 999 1000 void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) 1001 { 1002 int relalign = 0; 1003 1004 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { 1005 int size = sizeof(unsigned long); 1006 int d = (((unsigned long)dst ^ (unsigned long)src1) | 1007 ((unsigned long)dst ^ (unsigned long)src2)) & 1008 (size - 1); 1009 1010 relalign = d ? 1 << __ffs(d) : size; 1011 1012 /* 1013 * If we care about alignment, process as many bytes as 1014 * needed to advance dst and src to values whose alignments 1015 * equal their relative alignment. This will allow us to 1016 * process the remainder of the input using optimal strides. 1017 */ 1018 while (((unsigned long)dst & (relalign - 1)) && len > 0) { 1019 *dst++ = *src1++ ^ *src2++; 1020 len--; 1021 } 1022 } 1023 1024 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { 1025 *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; 1026 dst += 8; 1027 src1 += 8; 1028 src2 += 8; 1029 len -= 8; 1030 } 1031 1032 while (len >= 4 && !(relalign & 3)) { 1033 *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; 1034 dst += 4; 1035 src1 += 4; 1036 src2 += 4; 1037 len -= 4; 1038 } 1039 1040 while (len >= 2 && !(relalign & 1)) { 1041 *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; 1042 dst += 2; 1043 src1 += 2; 1044 src2 += 2; 1045 len -= 2; 1046 } 1047 1048 while (len--) 1049 *dst++ = *src1++ ^ *src2++; 1050 } 1051 EXPORT_SYMBOL_GPL(__crypto_xor); 1052 1053 unsigned int crypto_alg_extsize(struct crypto_alg *alg) 1054 { 1055 return alg->cra_ctxsize + 1056 (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1)); 1057 } 1058 EXPORT_SYMBOL_GPL(crypto_alg_extsize); 1059 1060 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend, 1061 u32 type, u32 mask) 1062 { 1063 int ret = 0; 1064 struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask); 1065 1066 if (!IS_ERR(alg)) { 1067 crypto_mod_put(alg); 1068 ret = 1; 1069 } 1070 1071 return ret; 1072 } 1073 EXPORT_SYMBOL_GPL(crypto_type_has_alg); 1074 1075 #ifdef CONFIG_CRYPTO_STATS 1076 void crypto_stats_get(struct crypto_alg *alg) 1077 { 1078 crypto_alg_get(alg); 1079 } 1080 EXPORT_SYMBOL_GPL(crypto_stats_get); 1081 1082 void crypto_stats_ablkcipher_encrypt(unsigned int nbytes, int ret, 1083 struct crypto_alg *alg) 1084 { 1085 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1086 atomic64_inc(&alg->stats.cipher.err_cnt); 1087 } else { 1088 atomic64_inc(&alg->stats.cipher.encrypt_cnt); 1089 atomic64_add(nbytes, &alg->stats.cipher.encrypt_tlen); 1090 } 1091 crypto_alg_put(alg); 1092 } 1093 EXPORT_SYMBOL_GPL(crypto_stats_ablkcipher_encrypt); 1094 1095 void crypto_stats_ablkcipher_decrypt(unsigned int nbytes, int ret, 1096 struct crypto_alg *alg) 1097 { 1098 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1099 atomic64_inc(&alg->stats.cipher.err_cnt); 1100 } else { 1101 atomic64_inc(&alg->stats.cipher.decrypt_cnt); 1102 atomic64_add(nbytes, &alg->stats.cipher.decrypt_tlen); 1103 } 1104 crypto_alg_put(alg); 1105 } 1106 EXPORT_SYMBOL_GPL(crypto_stats_ablkcipher_decrypt); 1107 1108 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg, 1109 int ret) 1110 { 1111 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1112 atomic64_inc(&alg->stats.aead.err_cnt); 1113 } else { 1114 atomic64_inc(&alg->stats.aead.encrypt_cnt); 1115 atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen); 1116 } 1117 crypto_alg_put(alg); 1118 } 1119 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt); 1120 1121 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg, 1122 int ret) 1123 { 1124 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1125 atomic64_inc(&alg->stats.aead.err_cnt); 1126 } else { 1127 atomic64_inc(&alg->stats.aead.decrypt_cnt); 1128 atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen); 1129 } 1130 crypto_alg_put(alg); 1131 } 1132 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt); 1133 1134 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret, 1135 struct crypto_alg *alg) 1136 { 1137 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1138 atomic64_inc(&alg->stats.akcipher.err_cnt); 1139 } else { 1140 atomic64_inc(&alg->stats.akcipher.encrypt_cnt); 1141 atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen); 1142 } 1143 crypto_alg_put(alg); 1144 } 1145 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt); 1146 1147 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret, 1148 struct crypto_alg *alg) 1149 { 1150 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1151 atomic64_inc(&alg->stats.akcipher.err_cnt); 1152 } else { 1153 atomic64_inc(&alg->stats.akcipher.decrypt_cnt); 1154 atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen); 1155 } 1156 crypto_alg_put(alg); 1157 } 1158 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt); 1159 1160 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg) 1161 { 1162 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1163 atomic64_inc(&alg->stats.akcipher.err_cnt); 1164 else 1165 atomic64_inc(&alg->stats.akcipher.sign_cnt); 1166 crypto_alg_put(alg); 1167 } 1168 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign); 1169 1170 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg) 1171 { 1172 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1173 atomic64_inc(&alg->stats.akcipher.err_cnt); 1174 else 1175 atomic64_inc(&alg->stats.akcipher.verify_cnt); 1176 crypto_alg_put(alg); 1177 } 1178 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify); 1179 1180 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg) 1181 { 1182 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1183 atomic64_inc(&alg->stats.compress.err_cnt); 1184 } else { 1185 atomic64_inc(&alg->stats.compress.compress_cnt); 1186 atomic64_add(slen, &alg->stats.compress.compress_tlen); 1187 } 1188 crypto_alg_put(alg); 1189 } 1190 EXPORT_SYMBOL_GPL(crypto_stats_compress); 1191 1192 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg) 1193 { 1194 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1195 atomic64_inc(&alg->stats.compress.err_cnt); 1196 } else { 1197 atomic64_inc(&alg->stats.compress.decompress_cnt); 1198 atomic64_add(slen, &alg->stats.compress.decompress_tlen); 1199 } 1200 crypto_alg_put(alg); 1201 } 1202 EXPORT_SYMBOL_GPL(crypto_stats_decompress); 1203 1204 void crypto_stats_ahash_update(unsigned int nbytes, int ret, 1205 struct crypto_alg *alg) 1206 { 1207 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1208 atomic64_inc(&alg->stats.hash.err_cnt); 1209 else 1210 atomic64_add(nbytes, &alg->stats.hash.hash_tlen); 1211 crypto_alg_put(alg); 1212 } 1213 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update); 1214 1215 void crypto_stats_ahash_final(unsigned int nbytes, int ret, 1216 struct crypto_alg *alg) 1217 { 1218 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1219 atomic64_inc(&alg->stats.hash.err_cnt); 1220 } else { 1221 atomic64_inc(&alg->stats.hash.hash_cnt); 1222 atomic64_add(nbytes, &alg->stats.hash.hash_tlen); 1223 } 1224 crypto_alg_put(alg); 1225 } 1226 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final); 1227 1228 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret) 1229 { 1230 if (ret) 1231 atomic64_inc(&alg->stats.kpp.err_cnt); 1232 else 1233 atomic64_inc(&alg->stats.kpp.setsecret_cnt); 1234 crypto_alg_put(alg); 1235 } 1236 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret); 1237 1238 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret) 1239 { 1240 if (ret) 1241 atomic64_inc(&alg->stats.kpp.err_cnt); 1242 else 1243 atomic64_inc(&alg->stats.kpp.generate_public_key_cnt); 1244 crypto_alg_put(alg); 1245 } 1246 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key); 1247 1248 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret) 1249 { 1250 if (ret) 1251 atomic64_inc(&alg->stats.kpp.err_cnt); 1252 else 1253 atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt); 1254 crypto_alg_put(alg); 1255 } 1256 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret); 1257 1258 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret) 1259 { 1260 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1261 atomic64_inc(&alg->stats.rng.err_cnt); 1262 else 1263 atomic64_inc(&alg->stats.rng.seed_cnt); 1264 crypto_alg_put(alg); 1265 } 1266 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed); 1267 1268 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen, 1269 int ret) 1270 { 1271 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1272 atomic64_inc(&alg->stats.rng.err_cnt); 1273 } else { 1274 atomic64_inc(&alg->stats.rng.generate_cnt); 1275 atomic64_add(dlen, &alg->stats.rng.generate_tlen); 1276 } 1277 crypto_alg_put(alg); 1278 } 1279 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate); 1280 1281 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret, 1282 struct crypto_alg *alg) 1283 { 1284 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1285 atomic64_inc(&alg->stats.cipher.err_cnt); 1286 } else { 1287 atomic64_inc(&alg->stats.cipher.encrypt_cnt); 1288 atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen); 1289 } 1290 crypto_alg_put(alg); 1291 } 1292 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt); 1293 1294 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret, 1295 struct crypto_alg *alg) 1296 { 1297 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1298 atomic64_inc(&alg->stats.cipher.err_cnt); 1299 } else { 1300 atomic64_inc(&alg->stats.cipher.decrypt_cnt); 1301 atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen); 1302 } 1303 crypto_alg_put(alg); 1304 } 1305 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt); 1306 #endif 1307 1308 static int __init crypto_algapi_init(void) 1309 { 1310 crypto_init_proc(); 1311 return 0; 1312 } 1313 1314 static void __exit crypto_algapi_exit(void) 1315 { 1316 crypto_exit_proc(); 1317 } 1318 1319 module_init(crypto_algapi_init); 1320 module_exit(crypto_algapi_exit); 1321 1322 MODULE_LICENSE("GPL"); 1323 MODULE_DESCRIPTION("Cryptographic algorithms API"); 1324