1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Asynchronous Cryptographic Hash operations. 4 * 5 * This is the implementation of the ahash (asynchronous hash) API. It differs 6 * from shash (synchronous hash) in that ahash supports asynchronous operations, 7 * and it hashes data from scatterlists instead of virtually addressed buffers. 8 * 9 * The ahash API provides access to both ahash and shash algorithms. The shash 10 * API only provides access to shash algorithms. 11 * 12 * Copyright (c) 2008 Loc Ho <lho@amcc.com> 13 */ 14 15 #include <crypto/scatterwalk.h> 16 #include <linux/cryptouser.h> 17 #include <linux/err.h> 18 #include <linux/kernel.h> 19 #include <linux/mm.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/slab.h> 23 #include <linux/seq_file.h> 24 #include <linux/string.h> 25 #include <linux/string_choices.h> 26 #include <net/netlink.h> 27 28 #include "hash.h" 29 30 #define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e 31 32 struct crypto_hash_walk { 33 const char *data; 34 35 unsigned int offset; 36 unsigned int flags; 37 38 struct page *pg; 39 unsigned int entrylen; 40 41 unsigned int total; 42 struct scatterlist *sg; 43 }; 44 45 static int ahash_def_finup(struct ahash_request *req); 46 47 static inline bool crypto_ahash_block_only(struct crypto_ahash *tfm) 48 { 49 return crypto_ahash_alg(tfm)->halg.base.cra_flags & 50 CRYPTO_AHASH_ALG_BLOCK_ONLY; 51 } 52 53 static inline bool crypto_ahash_final_nonzero(struct crypto_ahash *tfm) 54 { 55 return crypto_ahash_alg(tfm)->halg.base.cra_flags & 56 CRYPTO_AHASH_ALG_FINAL_NONZERO; 57 } 58 59 static inline bool crypto_ahash_need_fallback(struct crypto_ahash *tfm) 60 { 61 return crypto_ahash_alg(tfm)->halg.base.cra_flags & 62 CRYPTO_ALG_NEED_FALLBACK; 63 } 64 65 static inline void ahash_op_done(void *data, int err, 66 int (*finish)(struct ahash_request *, int)) 67 { 68 struct ahash_request *areq = data; 69 crypto_completion_t compl; 70 71 compl = areq->saved_complete; 72 data = areq->saved_data; 73 if (err == -EINPROGRESS) 74 goto out; 75 76 areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 77 78 err = finish(areq, err); 79 if (err == -EINPROGRESS || err == -EBUSY) 80 return; 81 82 out: 83 compl(data, err); 84 } 85 86 static int hash_walk_next(struct crypto_hash_walk *walk) 87 { 88 unsigned int offset = walk->offset; 89 unsigned int nbytes = min(walk->entrylen, 90 ((unsigned int)(PAGE_SIZE)) - offset); 91 92 walk->data = kmap_local_page(walk->pg); 93 walk->data += offset; 94 walk->entrylen -= nbytes; 95 return nbytes; 96 } 97 98 static int hash_walk_new_entry(struct crypto_hash_walk *walk) 99 { 100 struct scatterlist *sg; 101 102 sg = walk->sg; 103 walk->offset = sg->offset; 104 walk->pg = nth_page(sg_page(walk->sg), (walk->offset >> PAGE_SHIFT)); 105 walk->offset = offset_in_page(walk->offset); 106 walk->entrylen = sg->length; 107 108 if (walk->entrylen > walk->total) 109 walk->entrylen = walk->total; 110 walk->total -= walk->entrylen; 111 112 return hash_walk_next(walk); 113 } 114 115 static int crypto_hash_walk_first(struct ahash_request *req, 116 struct crypto_hash_walk *walk) 117 { 118 walk->total = req->nbytes; 119 walk->entrylen = 0; 120 121 if (!walk->total) 122 return 0; 123 124 walk->flags = req->base.flags; 125 126 if (ahash_request_isvirt(req)) { 127 walk->data = req->svirt; 128 walk->total = 0; 129 return req->nbytes; 130 } 131 132 walk->sg = req->src; 133 134 return hash_walk_new_entry(walk); 135 } 136 137 static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) 138 { 139 if ((walk->flags & CRYPTO_AHASH_REQ_VIRT)) 140 return err; 141 142 walk->data -= walk->offset; 143 144 kunmap_local(walk->data); 145 crypto_yield(walk->flags); 146 147 if (err) 148 return err; 149 150 if (walk->entrylen) { 151 walk->offset = 0; 152 walk->pg++; 153 return hash_walk_next(walk); 154 } 155 156 if (!walk->total) 157 return 0; 158 159 walk->sg = sg_next(walk->sg); 160 161 return hash_walk_new_entry(walk); 162 } 163 164 static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk) 165 { 166 return !(walk->entrylen | walk->total); 167 } 168 169 /* 170 * For an ahash tfm that is using an shash algorithm (instead of an ahash 171 * algorithm), this returns the underlying shash tfm. 172 */ 173 static inline struct crypto_shash *ahash_to_shash(struct crypto_ahash *tfm) 174 { 175 return *(struct crypto_shash **)crypto_ahash_ctx(tfm); 176 } 177 178 static inline struct shash_desc *prepare_shash_desc(struct ahash_request *req, 179 struct crypto_ahash *tfm) 180 { 181 struct shash_desc *desc = ahash_request_ctx(req); 182 183 desc->tfm = ahash_to_shash(tfm); 184 return desc; 185 } 186 187 int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc) 188 { 189 struct crypto_hash_walk walk; 190 int nbytes; 191 192 for (nbytes = crypto_hash_walk_first(req, &walk); nbytes > 0; 193 nbytes = crypto_hash_walk_done(&walk, nbytes)) 194 nbytes = crypto_shash_update(desc, walk.data, nbytes); 195 196 return nbytes; 197 } 198 EXPORT_SYMBOL_GPL(shash_ahash_update); 199 200 int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc) 201 { 202 struct crypto_hash_walk walk; 203 int nbytes; 204 205 nbytes = crypto_hash_walk_first(req, &walk); 206 if (!nbytes) 207 return crypto_shash_final(desc, req->result); 208 209 do { 210 nbytes = crypto_hash_walk_last(&walk) ? 211 crypto_shash_finup(desc, walk.data, nbytes, 212 req->result) : 213 crypto_shash_update(desc, walk.data, nbytes); 214 nbytes = crypto_hash_walk_done(&walk, nbytes); 215 } while (nbytes > 0); 216 217 return nbytes; 218 } 219 EXPORT_SYMBOL_GPL(shash_ahash_finup); 220 221 int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) 222 { 223 unsigned int nbytes = req->nbytes; 224 struct scatterlist *sg; 225 unsigned int offset; 226 struct page *page; 227 const u8 *data; 228 int err; 229 230 data = req->svirt; 231 if (!nbytes || ahash_request_isvirt(req)) 232 return crypto_shash_digest(desc, data, nbytes, req->result); 233 234 sg = req->src; 235 if (nbytes > sg->length) 236 return crypto_shash_init(desc) ?: 237 shash_ahash_finup(req, desc); 238 239 page = sg_page(sg); 240 offset = sg->offset; 241 data = lowmem_page_address(page) + offset; 242 if (!IS_ENABLED(CONFIG_HIGHMEM)) 243 return crypto_shash_digest(desc, data, nbytes, req->result); 244 245 page = nth_page(page, offset >> PAGE_SHIFT); 246 offset = offset_in_page(offset); 247 248 if (nbytes > (unsigned int)PAGE_SIZE - offset) 249 return crypto_shash_init(desc) ?: 250 shash_ahash_finup(req, desc); 251 252 data = kmap_local_page(page); 253 err = crypto_shash_digest(desc, data + offset, nbytes, 254 req->result); 255 kunmap_local(data); 256 return err; 257 } 258 EXPORT_SYMBOL_GPL(shash_ahash_digest); 259 260 static void crypto_exit_ahash_using_shash(struct crypto_tfm *tfm) 261 { 262 struct crypto_shash **ctx = crypto_tfm_ctx(tfm); 263 264 crypto_free_shash(*ctx); 265 } 266 267 static int crypto_init_ahash_using_shash(struct crypto_tfm *tfm) 268 { 269 struct crypto_alg *calg = tfm->__crt_alg; 270 struct crypto_ahash *crt = __crypto_ahash_cast(tfm); 271 struct crypto_shash **ctx = crypto_tfm_ctx(tfm); 272 struct crypto_shash *shash; 273 274 if (!crypto_mod_get(calg)) 275 return -EAGAIN; 276 277 shash = crypto_create_tfm(calg, &crypto_shash_type); 278 if (IS_ERR(shash)) { 279 crypto_mod_put(calg); 280 return PTR_ERR(shash); 281 } 282 283 crt->using_shash = true; 284 *ctx = shash; 285 tfm->exit = crypto_exit_ahash_using_shash; 286 287 crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) & 288 CRYPTO_TFM_NEED_KEY); 289 290 return 0; 291 } 292 293 static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, 294 unsigned int keylen) 295 { 296 return -ENOSYS; 297 } 298 299 static void ahash_set_needkey(struct crypto_ahash *tfm, struct ahash_alg *alg) 300 { 301 if (alg->setkey != ahash_nosetkey && 302 !(alg->halg.base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY)) 303 crypto_ahash_set_flags(tfm, CRYPTO_TFM_NEED_KEY); 304 } 305 306 int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 307 unsigned int keylen) 308 { 309 if (likely(tfm->using_shash)) { 310 struct crypto_shash *shash = ahash_to_shash(tfm); 311 int err; 312 313 err = crypto_shash_setkey(shash, key, keylen); 314 if (unlikely(err)) { 315 crypto_ahash_set_flags(tfm, 316 crypto_shash_get_flags(shash) & 317 CRYPTO_TFM_NEED_KEY); 318 return err; 319 } 320 } else { 321 struct ahash_alg *alg = crypto_ahash_alg(tfm); 322 int err; 323 324 err = alg->setkey(tfm, key, keylen); 325 if (!err && crypto_ahash_need_fallback(tfm)) 326 err = crypto_ahash_setkey(crypto_ahash_fb(tfm), 327 key, keylen); 328 if (unlikely(err)) { 329 ahash_set_needkey(tfm, alg); 330 return err; 331 } 332 } 333 crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); 334 return 0; 335 } 336 EXPORT_SYMBOL_GPL(crypto_ahash_setkey); 337 338 static int ahash_do_req_chain(struct ahash_request *req, 339 int (*const *op)(struct ahash_request *req)) 340 { 341 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 342 int err; 343 344 if (crypto_ahash_req_virt(tfm) || !ahash_request_isvirt(req)) 345 return (*op)(req); 346 347 if (crypto_ahash_statesize(tfm) > HASH_MAX_STATESIZE) 348 return -ENOSYS; 349 350 if (!crypto_ahash_need_fallback(tfm)) 351 return -ENOSYS; 352 353 if (crypto_hash_no_export_core(tfm)) 354 return -ENOSYS; 355 356 { 357 u8 state[HASH_MAX_STATESIZE]; 358 359 if (op == &crypto_ahash_alg(tfm)->digest) { 360 ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); 361 err = crypto_ahash_digest(req); 362 goto out_no_state; 363 } 364 365 err = crypto_ahash_export(req, state); 366 ahash_request_set_tfm(req, crypto_ahash_fb(tfm)); 367 err = err ?: crypto_ahash_import(req, state); 368 369 if (op == &crypto_ahash_alg(tfm)->finup) { 370 err = err ?: crypto_ahash_finup(req); 371 goto out_no_state; 372 } 373 374 err = err ?: 375 crypto_ahash_update(req) ?: 376 crypto_ahash_export(req, state); 377 378 ahash_request_set_tfm(req, tfm); 379 return err ?: crypto_ahash_import(req, state); 380 381 out_no_state: 382 ahash_request_set_tfm(req, tfm); 383 return err; 384 } 385 } 386 387 int crypto_ahash_init(struct ahash_request *req) 388 { 389 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 390 391 if (likely(tfm->using_shash)) 392 return crypto_shash_init(prepare_shash_desc(req, tfm)); 393 if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) 394 return -ENOKEY; 395 if (ahash_req_on_stack(req) && ahash_is_async(tfm)) 396 return -EAGAIN; 397 if (crypto_ahash_block_only(tfm)) { 398 u8 *buf = ahash_request_ctx(req); 399 400 buf += crypto_ahash_reqsize(tfm) - 1; 401 *buf = 0; 402 } 403 return crypto_ahash_alg(tfm)->init(req); 404 } 405 EXPORT_SYMBOL_GPL(crypto_ahash_init); 406 407 static void ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) 408 { 409 req->saved_complete = req->base.complete; 410 req->saved_data = req->base.data; 411 req->base.complete = cplt; 412 req->base.data = req; 413 } 414 415 static void ahash_restore_req(struct ahash_request *req) 416 { 417 req->base.complete = req->saved_complete; 418 req->base.data = req->saved_data; 419 } 420 421 static int ahash_update_finish(struct ahash_request *req, int err) 422 { 423 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 424 bool nonzero = crypto_ahash_final_nonzero(tfm); 425 int bs = crypto_ahash_blocksize(tfm); 426 u8 *blenp = ahash_request_ctx(req); 427 int blen; 428 u8 *buf; 429 430 blenp += crypto_ahash_reqsize(tfm) - 1; 431 blen = *blenp; 432 buf = blenp - bs; 433 434 if (blen) { 435 req->src = req->sg_head + 1; 436 if (sg_is_chain(req->src)) 437 req->src = sg_chain_ptr(req->src); 438 } 439 440 req->nbytes += nonzero - blen; 441 442 blen = err < 0 ? 0 : err + nonzero; 443 if (ahash_request_isvirt(req)) 444 memcpy(buf, req->svirt + req->nbytes - blen, blen); 445 else 446 memcpy_from_sglist(buf, req->src, req->nbytes - blen, blen); 447 *blenp = blen; 448 449 ahash_restore_req(req); 450 451 return err; 452 } 453 454 static void ahash_update_done(void *data, int err) 455 { 456 ahash_op_done(data, err, ahash_update_finish); 457 } 458 459 int crypto_ahash_update(struct ahash_request *req) 460 { 461 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 462 bool nonzero = crypto_ahash_final_nonzero(tfm); 463 int bs = crypto_ahash_blocksize(tfm); 464 u8 *blenp = ahash_request_ctx(req); 465 int blen, err; 466 u8 *buf; 467 468 if (likely(tfm->using_shash)) 469 return shash_ahash_update(req, ahash_request_ctx(req)); 470 if (ahash_req_on_stack(req) && ahash_is_async(tfm)) 471 return -EAGAIN; 472 if (!crypto_ahash_block_only(tfm)) 473 return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update); 474 475 blenp += crypto_ahash_reqsize(tfm) - 1; 476 blen = *blenp; 477 buf = blenp - bs; 478 479 if (blen + req->nbytes < bs + nonzero) { 480 if (ahash_request_isvirt(req)) 481 memcpy(buf + blen, req->svirt, req->nbytes); 482 else 483 memcpy_from_sglist(buf + blen, req->src, 0, 484 req->nbytes); 485 486 *blenp += req->nbytes; 487 return 0; 488 } 489 490 if (blen) { 491 memset(req->sg_head, 0, sizeof(req->sg_head[0])); 492 sg_set_buf(req->sg_head, buf, blen); 493 if (req->src != req->sg_head + 1) 494 sg_chain(req->sg_head, 2, req->src); 495 req->src = req->sg_head; 496 req->nbytes += blen; 497 } 498 req->nbytes -= nonzero; 499 500 ahash_save_req(req, ahash_update_done); 501 502 err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update); 503 if (err == -EINPROGRESS || err == -EBUSY) 504 return err; 505 506 return ahash_update_finish(req, err); 507 } 508 EXPORT_SYMBOL_GPL(crypto_ahash_update); 509 510 static int ahash_finup_finish(struct ahash_request *req, int err) 511 { 512 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 513 u8 *blenp = ahash_request_ctx(req); 514 int blen; 515 516 blenp += crypto_ahash_reqsize(tfm) - 1; 517 blen = *blenp; 518 519 if (blen) { 520 if (sg_is_last(req->src)) 521 req->src = NULL; 522 else { 523 req->src = req->sg_head + 1; 524 if (sg_is_chain(req->src)) 525 req->src = sg_chain_ptr(req->src); 526 } 527 req->nbytes -= blen; 528 } 529 530 ahash_restore_req(req); 531 532 return err; 533 } 534 535 static void ahash_finup_done(void *data, int err) 536 { 537 ahash_op_done(data, err, ahash_finup_finish); 538 } 539 540 int crypto_ahash_finup(struct ahash_request *req) 541 { 542 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 543 int bs = crypto_ahash_blocksize(tfm); 544 u8 *blenp = ahash_request_ctx(req); 545 int blen, err; 546 u8 *buf; 547 548 if (likely(tfm->using_shash)) 549 return shash_ahash_finup(req, ahash_request_ctx(req)); 550 if (ahash_req_on_stack(req) && ahash_is_async(tfm)) 551 return -EAGAIN; 552 if (!crypto_ahash_alg(tfm)->finup) 553 return ahash_def_finup(req); 554 if (!crypto_ahash_block_only(tfm)) 555 return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup); 556 557 blenp += crypto_ahash_reqsize(tfm) - 1; 558 blen = *blenp; 559 buf = blenp - bs; 560 561 if (blen) { 562 memset(req->sg_head, 0, sizeof(req->sg_head[0])); 563 sg_set_buf(req->sg_head, buf, blen); 564 if (!req->src) 565 sg_mark_end(req->sg_head); 566 else if (req->src != req->sg_head + 1) 567 sg_chain(req->sg_head, 2, req->src); 568 req->src = req->sg_head; 569 req->nbytes += blen; 570 } 571 572 ahash_save_req(req, ahash_finup_done); 573 574 err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup); 575 if (err == -EINPROGRESS || err == -EBUSY) 576 return err; 577 578 return ahash_finup_finish(req, err); 579 } 580 EXPORT_SYMBOL_GPL(crypto_ahash_finup); 581 582 int crypto_ahash_digest(struct ahash_request *req) 583 { 584 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 585 586 if (likely(tfm->using_shash)) 587 return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); 588 if (ahash_req_on_stack(req) && ahash_is_async(tfm)) 589 return -EAGAIN; 590 if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) 591 return -ENOKEY; 592 return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->digest); 593 } 594 EXPORT_SYMBOL_GPL(crypto_ahash_digest); 595 596 static void ahash_def_finup_done2(void *data, int err) 597 { 598 struct ahash_request *areq = data; 599 600 if (err == -EINPROGRESS) 601 return; 602 603 ahash_restore_req(areq); 604 ahash_request_complete(areq, err); 605 } 606 607 static int ahash_def_finup_finish1(struct ahash_request *req, int err) 608 { 609 if (err) 610 goto out; 611 612 req->base.complete = ahash_def_finup_done2; 613 614 err = crypto_ahash_final(req); 615 if (err == -EINPROGRESS || err == -EBUSY) 616 return err; 617 618 out: 619 ahash_restore_req(req); 620 return err; 621 } 622 623 static void ahash_def_finup_done1(void *data, int err) 624 { 625 ahash_op_done(data, err, ahash_def_finup_finish1); 626 } 627 628 static int ahash_def_finup(struct ahash_request *req) 629 { 630 int err; 631 632 ahash_save_req(req, ahash_def_finup_done1); 633 634 err = crypto_ahash_update(req); 635 if (err == -EINPROGRESS || err == -EBUSY) 636 return err; 637 638 return ahash_def_finup_finish1(req, err); 639 } 640 641 int crypto_ahash_export_core(struct ahash_request *req, void *out) 642 { 643 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 644 645 if (likely(tfm->using_shash)) 646 return crypto_shash_export_core(ahash_request_ctx(req), out); 647 return crypto_ahash_alg(tfm)->export_core(req, out); 648 } 649 EXPORT_SYMBOL_GPL(crypto_ahash_export_core); 650 651 int crypto_ahash_export(struct ahash_request *req, void *out) 652 { 653 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 654 655 if (likely(tfm->using_shash)) 656 return crypto_shash_export(ahash_request_ctx(req), out); 657 if (crypto_ahash_block_only(tfm)) { 658 unsigned int plen = crypto_ahash_blocksize(tfm) + 1; 659 unsigned int reqsize = crypto_ahash_reqsize(tfm); 660 unsigned int ss = crypto_ahash_statesize(tfm); 661 u8 *buf = ahash_request_ctx(req); 662 663 memcpy(out + ss - plen, buf + reqsize - plen, plen); 664 } 665 return crypto_ahash_alg(tfm)->export(req, out); 666 } 667 EXPORT_SYMBOL_GPL(crypto_ahash_export); 668 669 int crypto_ahash_import_core(struct ahash_request *req, const void *in) 670 { 671 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 672 673 if (likely(tfm->using_shash)) 674 return crypto_shash_import_core(prepare_shash_desc(req, tfm), 675 in); 676 if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) 677 return -ENOKEY; 678 return crypto_ahash_alg(tfm)->import_core(req, in); 679 } 680 EXPORT_SYMBOL_GPL(crypto_ahash_import_core); 681 682 int crypto_ahash_import(struct ahash_request *req, const void *in) 683 { 684 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 685 686 if (likely(tfm->using_shash)) 687 return crypto_shash_import(prepare_shash_desc(req, tfm), in); 688 if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) 689 return -ENOKEY; 690 if (crypto_ahash_block_only(tfm)) { 691 unsigned int reqsize = crypto_ahash_reqsize(tfm); 692 u8 *buf = ahash_request_ctx(req); 693 694 buf[reqsize - 1] = 0; 695 } 696 return crypto_ahash_alg(tfm)->import(req, in); 697 } 698 EXPORT_SYMBOL_GPL(crypto_ahash_import); 699 700 static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) 701 { 702 struct crypto_ahash *hash = __crypto_ahash_cast(tfm); 703 struct ahash_alg *alg = crypto_ahash_alg(hash); 704 705 if (alg->exit_tfm) 706 alg->exit_tfm(hash); 707 else if (tfm->__crt_alg->cra_exit) 708 tfm->__crt_alg->cra_exit(tfm); 709 710 if (crypto_ahash_need_fallback(hash)) 711 crypto_free_ahash(crypto_ahash_fb(hash)); 712 } 713 714 static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) 715 { 716 struct crypto_ahash *hash = __crypto_ahash_cast(tfm); 717 struct ahash_alg *alg = crypto_ahash_alg(hash); 718 struct crypto_ahash *fb = NULL; 719 int err; 720 721 crypto_ahash_set_statesize(hash, alg->halg.statesize); 722 crypto_ahash_set_reqsize(hash, crypto_tfm_alg_reqsize(tfm)); 723 724 if (tfm->__crt_alg->cra_type == &crypto_shash_type) 725 return crypto_init_ahash_using_shash(tfm); 726 727 if (crypto_ahash_need_fallback(hash)) { 728 fb = crypto_alloc_ahash(crypto_ahash_alg_name(hash), 729 CRYPTO_ALG_REQ_VIRT, 730 CRYPTO_ALG_ASYNC | 731 CRYPTO_ALG_REQ_VIRT | 732 CRYPTO_AHASH_ALG_NO_EXPORT_CORE); 733 if (IS_ERR(fb)) 734 return PTR_ERR(fb); 735 736 tfm->fb = crypto_ahash_tfm(fb); 737 } 738 739 ahash_set_needkey(hash, alg); 740 741 tfm->exit = crypto_ahash_exit_tfm; 742 743 if (alg->init_tfm) 744 err = alg->init_tfm(hash); 745 else if (tfm->__crt_alg->cra_init) 746 err = tfm->__crt_alg->cra_init(tfm); 747 else 748 return 0; 749 750 if (err) 751 goto out_free_sync_hash; 752 753 if (!ahash_is_async(hash) && crypto_ahash_reqsize(hash) > 754 MAX_SYNC_HASH_REQSIZE) 755 goto out_exit_tfm; 756 757 BUILD_BUG_ON(HASH_MAX_DESCSIZE > MAX_SYNC_HASH_REQSIZE); 758 if (crypto_ahash_reqsize(hash) < HASH_MAX_DESCSIZE) 759 crypto_ahash_set_reqsize(hash, HASH_MAX_DESCSIZE); 760 761 return 0; 762 763 out_exit_tfm: 764 if (alg->exit_tfm) 765 alg->exit_tfm(hash); 766 else if (tfm->__crt_alg->cra_exit) 767 tfm->__crt_alg->cra_exit(tfm); 768 err = -EINVAL; 769 out_free_sync_hash: 770 crypto_free_ahash(fb); 771 return err; 772 } 773 774 static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) 775 { 776 if (alg->cra_type == &crypto_shash_type) 777 return sizeof(struct crypto_shash *); 778 779 return crypto_alg_extsize(alg); 780 } 781 782 static void crypto_ahash_free_instance(struct crypto_instance *inst) 783 { 784 struct ahash_instance *ahash = ahash_instance(inst); 785 786 ahash->free(ahash); 787 } 788 789 static int __maybe_unused crypto_ahash_report( 790 struct sk_buff *skb, struct crypto_alg *alg) 791 { 792 struct crypto_report_hash rhash; 793 794 memset(&rhash, 0, sizeof(rhash)); 795 796 strscpy(rhash.type, "ahash", sizeof(rhash.type)); 797 798 rhash.blocksize = alg->cra_blocksize; 799 rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize; 800 801 return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash); 802 } 803 804 static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 805 __maybe_unused; 806 static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 807 { 808 seq_printf(m, "type : ahash\n"); 809 seq_printf(m, "async : %s\n", 810 str_yes_no(alg->cra_flags & CRYPTO_ALG_ASYNC)); 811 seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); 812 seq_printf(m, "digestsize : %u\n", 813 __crypto_hash_alg_common(alg)->digestsize); 814 } 815 816 static const struct crypto_type crypto_ahash_type = { 817 .extsize = crypto_ahash_extsize, 818 .init_tfm = crypto_ahash_init_tfm, 819 .free = crypto_ahash_free_instance, 820 #ifdef CONFIG_PROC_FS 821 .show = crypto_ahash_show, 822 #endif 823 #if IS_ENABLED(CONFIG_CRYPTO_USER) 824 .report = crypto_ahash_report, 825 #endif 826 .maskclear = ~CRYPTO_ALG_TYPE_MASK, 827 .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, 828 .type = CRYPTO_ALG_TYPE_AHASH, 829 .tfmsize = offsetof(struct crypto_ahash, base), 830 .algsize = offsetof(struct ahash_alg, halg.base), 831 }; 832 833 int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, 834 struct crypto_instance *inst, 835 const char *name, u32 type, u32 mask) 836 { 837 spawn->base.frontend = &crypto_ahash_type; 838 return crypto_grab_spawn(&spawn->base, inst, name, type, mask); 839 } 840 EXPORT_SYMBOL_GPL(crypto_grab_ahash); 841 842 struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, 843 u32 mask) 844 { 845 return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask); 846 } 847 EXPORT_SYMBOL_GPL(crypto_alloc_ahash); 848 849 int crypto_has_ahash(const char *alg_name, u32 type, u32 mask) 850 { 851 return crypto_type_has_alg(alg_name, &crypto_ahash_type, type, mask); 852 } 853 EXPORT_SYMBOL_GPL(crypto_has_ahash); 854 855 bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg) 856 { 857 struct crypto_alg *alg = &halg->base; 858 859 if (alg->cra_type == &crypto_shash_type) 860 return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg)); 861 862 return __crypto_ahash_alg(alg)->setkey != ahash_nosetkey; 863 } 864 EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey); 865 866 struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash) 867 { 868 struct hash_alg_common *halg = crypto_hash_alg_common(hash); 869 struct crypto_tfm *tfm = crypto_ahash_tfm(hash); 870 struct crypto_ahash *fb = NULL; 871 struct crypto_ahash *nhash; 872 struct ahash_alg *alg; 873 int err; 874 875 if (!crypto_hash_alg_has_setkey(halg)) { 876 tfm = crypto_tfm_get(tfm); 877 if (IS_ERR(tfm)) 878 return ERR_CAST(tfm); 879 880 return hash; 881 } 882 883 nhash = crypto_clone_tfm(&crypto_ahash_type, tfm); 884 885 if (IS_ERR(nhash)) 886 return nhash; 887 888 nhash->reqsize = hash->reqsize; 889 nhash->statesize = hash->statesize; 890 891 if (likely(hash->using_shash)) { 892 struct crypto_shash **nctx = crypto_ahash_ctx(nhash); 893 struct crypto_shash *shash; 894 895 shash = crypto_clone_shash(ahash_to_shash(hash)); 896 if (IS_ERR(shash)) { 897 err = PTR_ERR(shash); 898 goto out_free_nhash; 899 } 900 crypto_ahash_tfm(nhash)->exit = crypto_exit_ahash_using_shash; 901 nhash->using_shash = true; 902 *nctx = shash; 903 return nhash; 904 } 905 906 if (crypto_ahash_need_fallback(hash)) { 907 fb = crypto_clone_ahash(crypto_ahash_fb(hash)); 908 err = PTR_ERR(fb); 909 if (IS_ERR(fb)) 910 goto out_free_nhash; 911 912 crypto_ahash_tfm(nhash)->fb = crypto_ahash_tfm(fb); 913 } 914 915 err = -ENOSYS; 916 alg = crypto_ahash_alg(hash); 917 if (!alg->clone_tfm) 918 goto out_free_fb; 919 920 err = alg->clone_tfm(nhash, hash); 921 if (err) 922 goto out_free_fb; 923 924 crypto_ahash_tfm(nhash)->exit = crypto_ahash_exit_tfm; 925 926 return nhash; 927 928 out_free_fb: 929 crypto_free_ahash(fb); 930 out_free_nhash: 931 crypto_free_ahash(nhash); 932 return ERR_PTR(err); 933 } 934 EXPORT_SYMBOL_GPL(crypto_clone_ahash); 935 936 static int ahash_default_export_core(struct ahash_request *req, void *out) 937 { 938 return -ENOSYS; 939 } 940 941 static int ahash_default_import_core(struct ahash_request *req, const void *in) 942 { 943 return -ENOSYS; 944 } 945 946 static int ahash_prepare_alg(struct ahash_alg *alg) 947 { 948 struct crypto_alg *base = &alg->halg.base; 949 int err; 950 951 if (alg->halg.statesize == 0) 952 return -EINVAL; 953 954 if (base->cra_reqsize && base->cra_reqsize < alg->halg.statesize) 955 return -EINVAL; 956 957 if (!(base->cra_flags & CRYPTO_ALG_ASYNC) && 958 base->cra_reqsize > MAX_SYNC_HASH_REQSIZE) 959 return -EINVAL; 960 961 if (base->cra_flags & CRYPTO_ALG_NEED_FALLBACK && 962 base->cra_flags & CRYPTO_ALG_NO_FALLBACK) 963 return -EINVAL; 964 965 err = hash_prepare_alg(&alg->halg); 966 if (err) 967 return err; 968 969 base->cra_type = &crypto_ahash_type; 970 base->cra_flags |= CRYPTO_ALG_TYPE_AHASH; 971 972 if ((base->cra_flags ^ CRYPTO_ALG_REQ_VIRT) & 973 (CRYPTO_ALG_ASYNC | CRYPTO_ALG_REQ_VIRT) && 974 !(base->cra_flags & CRYPTO_ALG_NO_FALLBACK)) 975 base->cra_flags |= CRYPTO_ALG_NEED_FALLBACK; 976 977 if (!alg->setkey) 978 alg->setkey = ahash_nosetkey; 979 980 if (base->cra_flags & CRYPTO_AHASH_ALG_BLOCK_ONLY) { 981 BUILD_BUG_ON(MAX_ALGAPI_BLOCKSIZE >= 256); 982 if (!alg->finup) 983 return -EINVAL; 984 985 base->cra_reqsize += base->cra_blocksize + 1; 986 alg->halg.statesize += base->cra_blocksize + 1; 987 alg->export_core = alg->export; 988 alg->import_core = alg->import; 989 } else if (!alg->export_core || !alg->import_core) { 990 alg->export_core = ahash_default_export_core; 991 alg->import_core = ahash_default_import_core; 992 base->cra_flags |= CRYPTO_AHASH_ALG_NO_EXPORT_CORE; 993 } 994 995 return 0; 996 } 997 998 int crypto_register_ahash(struct ahash_alg *alg) 999 { 1000 struct crypto_alg *base = &alg->halg.base; 1001 int err; 1002 1003 err = ahash_prepare_alg(alg); 1004 if (err) 1005 return err; 1006 1007 return crypto_register_alg(base); 1008 } 1009 EXPORT_SYMBOL_GPL(crypto_register_ahash); 1010 1011 void crypto_unregister_ahash(struct ahash_alg *alg) 1012 { 1013 crypto_unregister_alg(&alg->halg.base); 1014 } 1015 EXPORT_SYMBOL_GPL(crypto_unregister_ahash); 1016 1017 int crypto_register_ahashes(struct ahash_alg *algs, int count) 1018 { 1019 int i, ret; 1020 1021 for (i = 0; i < count; i++) { 1022 ret = crypto_register_ahash(&algs[i]); 1023 if (ret) 1024 goto err; 1025 } 1026 1027 return 0; 1028 1029 err: 1030 for (--i; i >= 0; --i) 1031 crypto_unregister_ahash(&algs[i]); 1032 1033 return ret; 1034 } 1035 EXPORT_SYMBOL_GPL(crypto_register_ahashes); 1036 1037 void crypto_unregister_ahashes(struct ahash_alg *algs, int count) 1038 { 1039 int i; 1040 1041 for (i = count - 1; i >= 0; --i) 1042 crypto_unregister_ahash(&algs[i]); 1043 } 1044 EXPORT_SYMBOL_GPL(crypto_unregister_ahashes); 1045 1046 int ahash_register_instance(struct crypto_template *tmpl, 1047 struct ahash_instance *inst) 1048 { 1049 int err; 1050 1051 if (WARN_ON(!inst->free)) 1052 return -EINVAL; 1053 1054 err = ahash_prepare_alg(&inst->alg); 1055 if (err) 1056 return err; 1057 1058 return crypto_register_instance(tmpl, ahash_crypto_instance(inst)); 1059 } 1060 EXPORT_SYMBOL_GPL(ahash_register_instance); 1061 1062 void ahash_request_free(struct ahash_request *req) 1063 { 1064 if (unlikely(!req)) 1065 return; 1066 1067 if (!ahash_req_on_stack(req)) { 1068 kfree(req); 1069 return; 1070 } 1071 1072 ahash_request_zero(req); 1073 } 1074 EXPORT_SYMBOL_GPL(ahash_request_free); 1075 1076 int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, 1077 unsigned int len, u8 *out) 1078 { 1079 HASH_REQUEST_ON_STACK(req, crypto_ahash_fb(tfm)); 1080 int err; 1081 1082 ahash_request_set_callback(req, 0, NULL, NULL); 1083 ahash_request_set_virt(req, data, out, len); 1084 err = crypto_ahash_digest(req); 1085 1086 ahash_request_zero(req); 1087 1088 return err; 1089 } 1090 EXPORT_SYMBOL_GPL(crypto_hash_digest); 1091 1092 void ahash_free_singlespawn_instance(struct ahash_instance *inst) 1093 { 1094 crypto_drop_spawn(ahash_instance_ctx(inst)); 1095 kfree(inst); 1096 } 1097 EXPORT_SYMBOL_GPL(ahash_free_singlespawn_instance); 1098 1099 MODULE_LICENSE("GPL"); 1100 MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); 1101