1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Asynchronous Cryptographic Hash operations. 4 * 5 * This is the asynchronous version of hash.c with notification of 6 * completion via a callback. 7 * 8 * Copyright (c) 2008 Loc Ho <lho@amcc.com> 9 */ 10 11 #include <crypto/internal/hash.h> 12 #include <crypto/scatterwalk.h> 13 #include <linux/err.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/sched.h> 17 #include <linux/slab.h> 18 #include <linux/seq_file.h> 19 #include <linux/cryptouser.h> 20 #include <linux/compiler.h> 21 #include <net/netlink.h> 22 23 #include "internal.h" 24 25 static const struct crypto_type crypto_ahash_type; 26 27 struct ahash_request_priv { 28 crypto_completion_t complete; 29 void *data; 30 u8 *result; 31 u32 flags; 32 void *ubuf[] CRYPTO_MINALIGN_ATTR; 33 }; 34 35 static inline struct ahash_alg *crypto_ahash_alg(struct crypto_ahash *hash) 36 { 37 return container_of(crypto_hash_alg_common(hash), struct ahash_alg, 38 halg); 39 } 40 41 static int hash_walk_next(struct crypto_hash_walk *walk) 42 { 43 unsigned int alignmask = walk->alignmask; 44 unsigned int offset = walk->offset; 45 unsigned int nbytes = min(walk->entrylen, 46 ((unsigned int)(PAGE_SIZE)) - offset); 47 48 walk->data = kmap_local_page(walk->pg); 49 walk->data += offset; 50 51 if (offset & alignmask) { 52 unsigned int unaligned = alignmask + 1 - (offset & alignmask); 53 54 if (nbytes > unaligned) 55 nbytes = unaligned; 56 } 57 58 walk->entrylen -= nbytes; 59 return nbytes; 60 } 61 62 static int hash_walk_new_entry(struct crypto_hash_walk *walk) 63 { 64 struct scatterlist *sg; 65 66 sg = walk->sg; 67 walk->offset = sg->offset; 68 walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); 69 walk->offset = offset_in_page(walk->offset); 70 walk->entrylen = sg->length; 71 72 if (walk->entrylen > walk->total) 73 walk->entrylen = walk->total; 74 walk->total -= walk->entrylen; 75 76 return hash_walk_next(walk); 77 } 78 79 int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) 80 { 81 unsigned int alignmask = walk->alignmask; 82 83 walk->data -= walk->offset; 84 85 if (walk->entrylen && (walk->offset & alignmask) && !err) { 86 unsigned int nbytes; 87 88 walk->offset = ALIGN(walk->offset, alignmask + 1); 89 nbytes = min(walk->entrylen, 90 (unsigned int)(PAGE_SIZE - walk->offset)); 91 if (nbytes) { 92 walk->entrylen -= nbytes; 93 walk->data += walk->offset; 94 return nbytes; 95 } 96 } 97 98 kunmap_local(walk->data); 99 crypto_yield(walk->flags); 100 101 if (err) 102 return err; 103 104 if (walk->entrylen) { 105 walk->offset = 0; 106 walk->pg++; 107 return hash_walk_next(walk); 108 } 109 110 if (!walk->total) 111 return 0; 112 113 walk->sg = sg_next(walk->sg); 114 115 return hash_walk_new_entry(walk); 116 } 117 EXPORT_SYMBOL_GPL(crypto_hash_walk_done); 118 119 int crypto_hash_walk_first(struct ahash_request *req, 120 struct crypto_hash_walk *walk) 121 { 122 walk->total = req->nbytes; 123 124 if (!walk->total) { 125 walk->entrylen = 0; 126 return 0; 127 } 128 129 walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req)); 130 walk->sg = req->src; 131 walk->flags = req->base.flags; 132 133 return hash_walk_new_entry(walk); 134 } 135 EXPORT_SYMBOL_GPL(crypto_hash_walk_first); 136 137 static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key, 138 unsigned int keylen) 139 { 140 unsigned long alignmask = crypto_ahash_alignmask(tfm); 141 int ret; 142 u8 *buffer, *alignbuffer; 143 unsigned long absize; 144 145 absize = keylen + alignmask; 146 buffer = kmalloc(absize, GFP_KERNEL); 147 if (!buffer) 148 return -ENOMEM; 149 150 alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); 151 memcpy(alignbuffer, key, keylen); 152 ret = tfm->setkey(tfm, alignbuffer, keylen); 153 kfree_sensitive(buffer); 154 return ret; 155 } 156 157 static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, 158 unsigned int keylen) 159 { 160 return -ENOSYS; 161 } 162 163 static void ahash_set_needkey(struct crypto_ahash *tfm) 164 { 165 const struct hash_alg_common *alg = crypto_hash_alg_common(tfm); 166 167 if (tfm->setkey != ahash_nosetkey && 168 !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY)) 169 crypto_ahash_set_flags(tfm, CRYPTO_TFM_NEED_KEY); 170 } 171 172 int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 173 unsigned int keylen) 174 { 175 unsigned long alignmask = crypto_ahash_alignmask(tfm); 176 int err; 177 178 if ((unsigned long)key & alignmask) 179 err = ahash_setkey_unaligned(tfm, key, keylen); 180 else 181 err = tfm->setkey(tfm, key, keylen); 182 183 if (unlikely(err)) { 184 ahash_set_needkey(tfm); 185 return err; 186 } 187 188 crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); 189 return 0; 190 } 191 EXPORT_SYMBOL_GPL(crypto_ahash_setkey); 192 193 static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt, 194 bool has_state) 195 { 196 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 197 unsigned long alignmask = crypto_ahash_alignmask(tfm); 198 unsigned int ds = crypto_ahash_digestsize(tfm); 199 struct ahash_request *subreq; 200 unsigned int subreq_size; 201 unsigned int reqsize; 202 u8 *result; 203 gfp_t gfp; 204 u32 flags; 205 206 subreq_size = sizeof(*subreq); 207 reqsize = crypto_ahash_reqsize(tfm); 208 reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment()); 209 subreq_size += reqsize; 210 subreq_size += ds; 211 subreq_size += alignmask & ~(crypto_tfm_ctx_alignment() - 1); 212 213 flags = ahash_request_flags(req); 214 gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; 215 subreq = kmalloc(subreq_size, gfp); 216 if (!subreq) 217 return -ENOMEM; 218 219 ahash_request_set_tfm(subreq, tfm); 220 ahash_request_set_callback(subreq, flags, cplt, req); 221 222 result = (u8 *)(subreq + 1) + reqsize; 223 result = PTR_ALIGN(result, alignmask + 1); 224 225 ahash_request_set_crypt(subreq, req->src, result, req->nbytes); 226 227 if (has_state) { 228 void *state; 229 230 state = kmalloc(crypto_ahash_statesize(tfm), gfp); 231 if (!state) { 232 kfree(subreq); 233 return -ENOMEM; 234 } 235 236 crypto_ahash_export(req, state); 237 crypto_ahash_import(subreq, state); 238 kfree_sensitive(state); 239 } 240 241 req->priv = subreq; 242 243 return 0; 244 } 245 246 static void ahash_restore_req(struct ahash_request *req, int err) 247 { 248 struct ahash_request *subreq = req->priv; 249 250 if (!err) 251 memcpy(req->result, subreq->result, 252 crypto_ahash_digestsize(crypto_ahash_reqtfm(req))); 253 254 req->priv = NULL; 255 256 kfree_sensitive(subreq); 257 } 258 259 static void ahash_op_unaligned_done(void *data, int err) 260 { 261 struct ahash_request *areq = data; 262 263 if (err == -EINPROGRESS) 264 goto out; 265 266 /* First copy req->result into req->priv.result */ 267 ahash_restore_req(areq, err); 268 269 out: 270 /* Complete the ORIGINAL request. */ 271 ahash_request_complete(areq, err); 272 } 273 274 static int ahash_op_unaligned(struct ahash_request *req, 275 int (*op)(struct ahash_request *), 276 bool has_state) 277 { 278 int err; 279 280 err = ahash_save_req(req, ahash_op_unaligned_done, has_state); 281 if (err) 282 return err; 283 284 err = op(req->priv); 285 if (err == -EINPROGRESS || err == -EBUSY) 286 return err; 287 288 ahash_restore_req(req, err); 289 290 return err; 291 } 292 293 static int crypto_ahash_op(struct ahash_request *req, 294 int (*op)(struct ahash_request *), 295 bool has_state) 296 { 297 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 298 unsigned long alignmask = crypto_ahash_alignmask(tfm); 299 300 if ((unsigned long)req->result & alignmask) 301 return ahash_op_unaligned(req, op, has_state); 302 303 return op(req); 304 } 305 306 int crypto_ahash_final(struct ahash_request *req) 307 { 308 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 309 struct crypto_alg *alg = tfm->base.__crt_alg; 310 unsigned int nbytes = req->nbytes; 311 int ret; 312 313 crypto_stats_get(alg); 314 ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final, true); 315 crypto_stats_ahash_final(nbytes, ret, alg); 316 return ret; 317 } 318 EXPORT_SYMBOL_GPL(crypto_ahash_final); 319 320 int crypto_ahash_finup(struct ahash_request *req) 321 { 322 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 323 struct crypto_alg *alg = tfm->base.__crt_alg; 324 unsigned int nbytes = req->nbytes; 325 int ret; 326 327 crypto_stats_get(alg); 328 ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup, true); 329 crypto_stats_ahash_final(nbytes, ret, alg); 330 return ret; 331 } 332 EXPORT_SYMBOL_GPL(crypto_ahash_finup); 333 334 int crypto_ahash_digest(struct ahash_request *req) 335 { 336 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 337 struct crypto_alg *alg = tfm->base.__crt_alg; 338 unsigned int nbytes = req->nbytes; 339 int ret; 340 341 crypto_stats_get(alg); 342 if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) 343 ret = -ENOKEY; 344 else 345 ret = crypto_ahash_op(req, tfm->digest, false); 346 crypto_stats_ahash_final(nbytes, ret, alg); 347 return ret; 348 } 349 EXPORT_SYMBOL_GPL(crypto_ahash_digest); 350 351 static void ahash_def_finup_done2(void *data, int err) 352 { 353 struct ahash_request *areq = data; 354 355 if (err == -EINPROGRESS) 356 return; 357 358 ahash_restore_req(areq, err); 359 360 ahash_request_complete(areq, err); 361 } 362 363 static int ahash_def_finup_finish1(struct ahash_request *req, int err) 364 { 365 struct ahash_request *subreq = req->priv; 366 367 if (err) 368 goto out; 369 370 subreq->base.complete = ahash_def_finup_done2; 371 372 err = crypto_ahash_reqtfm(req)->final(subreq); 373 if (err == -EINPROGRESS || err == -EBUSY) 374 return err; 375 376 out: 377 ahash_restore_req(req, err); 378 return err; 379 } 380 381 static void ahash_def_finup_done1(void *data, int err) 382 { 383 struct ahash_request *areq = data; 384 struct ahash_request *subreq; 385 386 if (err == -EINPROGRESS) 387 goto out; 388 389 subreq = areq->priv; 390 subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG; 391 392 err = ahash_def_finup_finish1(areq, err); 393 if (err == -EINPROGRESS || err == -EBUSY) 394 return; 395 396 out: 397 ahash_request_complete(areq, err); 398 } 399 400 static int ahash_def_finup(struct ahash_request *req) 401 { 402 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 403 int err; 404 405 err = ahash_save_req(req, ahash_def_finup_done1, true); 406 if (err) 407 return err; 408 409 err = tfm->update(req->priv); 410 if (err == -EINPROGRESS || err == -EBUSY) 411 return err; 412 413 return ahash_def_finup_finish1(req, err); 414 } 415 416 static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) 417 { 418 struct crypto_ahash *hash = __crypto_ahash_cast(tfm); 419 struct ahash_alg *alg = crypto_ahash_alg(hash); 420 421 alg->exit_tfm(hash); 422 } 423 424 static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) 425 { 426 struct crypto_ahash *hash = __crypto_ahash_cast(tfm); 427 struct ahash_alg *alg = crypto_ahash_alg(hash); 428 429 hash->setkey = ahash_nosetkey; 430 431 if (tfm->__crt_alg->cra_type != &crypto_ahash_type) 432 return crypto_init_shash_ops_async(tfm); 433 434 hash->init = alg->init; 435 hash->update = alg->update; 436 hash->final = alg->final; 437 hash->finup = alg->finup ?: ahash_def_finup; 438 hash->digest = alg->digest; 439 hash->export = alg->export; 440 hash->import = alg->import; 441 442 if (alg->setkey) { 443 hash->setkey = alg->setkey; 444 ahash_set_needkey(hash); 445 } 446 447 if (alg->exit_tfm) 448 tfm->exit = crypto_ahash_exit_tfm; 449 450 return alg->init_tfm ? alg->init_tfm(hash) : 0; 451 } 452 453 static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) 454 { 455 if (alg->cra_type != &crypto_ahash_type) 456 return sizeof(struct crypto_shash *); 457 458 return crypto_alg_extsize(alg); 459 } 460 461 static void crypto_ahash_free_instance(struct crypto_instance *inst) 462 { 463 struct ahash_instance *ahash = ahash_instance(inst); 464 465 ahash->free(ahash); 466 } 467 468 #ifdef CONFIG_NET 469 static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg) 470 { 471 struct crypto_report_hash rhash; 472 473 memset(&rhash, 0, sizeof(rhash)); 474 475 strscpy(rhash.type, "ahash", sizeof(rhash.type)); 476 477 rhash.blocksize = alg->cra_blocksize; 478 rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize; 479 480 return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash); 481 } 482 #else 483 static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg) 484 { 485 return -ENOSYS; 486 } 487 #endif 488 489 static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 490 __maybe_unused; 491 static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 492 { 493 seq_printf(m, "type : ahash\n"); 494 seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? 495 "yes" : "no"); 496 seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); 497 seq_printf(m, "digestsize : %u\n", 498 __crypto_hash_alg_common(alg)->digestsize); 499 } 500 501 static const struct crypto_type crypto_ahash_type = { 502 .extsize = crypto_ahash_extsize, 503 .init_tfm = crypto_ahash_init_tfm, 504 .free = crypto_ahash_free_instance, 505 #ifdef CONFIG_PROC_FS 506 .show = crypto_ahash_show, 507 #endif 508 .report = crypto_ahash_report, 509 .maskclear = ~CRYPTO_ALG_TYPE_MASK, 510 .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, 511 .type = CRYPTO_ALG_TYPE_AHASH, 512 .tfmsize = offsetof(struct crypto_ahash, base), 513 }; 514 515 int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, 516 struct crypto_instance *inst, 517 const char *name, u32 type, u32 mask) 518 { 519 spawn->base.frontend = &crypto_ahash_type; 520 return crypto_grab_spawn(&spawn->base, inst, name, type, mask); 521 } 522 EXPORT_SYMBOL_GPL(crypto_grab_ahash); 523 524 struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, 525 u32 mask) 526 { 527 return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask); 528 } 529 EXPORT_SYMBOL_GPL(crypto_alloc_ahash); 530 531 int crypto_has_ahash(const char *alg_name, u32 type, u32 mask) 532 { 533 return crypto_type_has_alg(alg_name, &crypto_ahash_type, type, mask); 534 } 535 EXPORT_SYMBOL_GPL(crypto_has_ahash); 536 537 static int ahash_prepare_alg(struct ahash_alg *alg) 538 { 539 struct crypto_alg *base = &alg->halg.base; 540 541 if (alg->halg.digestsize > HASH_MAX_DIGESTSIZE || 542 alg->halg.statesize > HASH_MAX_STATESIZE || 543 alg->halg.statesize == 0) 544 return -EINVAL; 545 546 base->cra_type = &crypto_ahash_type; 547 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; 548 base->cra_flags |= CRYPTO_ALG_TYPE_AHASH; 549 550 return 0; 551 } 552 553 int crypto_register_ahash(struct ahash_alg *alg) 554 { 555 struct crypto_alg *base = &alg->halg.base; 556 int err; 557 558 err = ahash_prepare_alg(alg); 559 if (err) 560 return err; 561 562 return crypto_register_alg(base); 563 } 564 EXPORT_SYMBOL_GPL(crypto_register_ahash); 565 566 void crypto_unregister_ahash(struct ahash_alg *alg) 567 { 568 crypto_unregister_alg(&alg->halg.base); 569 } 570 EXPORT_SYMBOL_GPL(crypto_unregister_ahash); 571 572 int crypto_register_ahashes(struct ahash_alg *algs, int count) 573 { 574 int i, ret; 575 576 for (i = 0; i < count; i++) { 577 ret = crypto_register_ahash(&algs[i]); 578 if (ret) 579 goto err; 580 } 581 582 return 0; 583 584 err: 585 for (--i; i >= 0; --i) 586 crypto_unregister_ahash(&algs[i]); 587 588 return ret; 589 } 590 EXPORT_SYMBOL_GPL(crypto_register_ahashes); 591 592 void crypto_unregister_ahashes(struct ahash_alg *algs, int count) 593 { 594 int i; 595 596 for (i = count - 1; i >= 0; --i) 597 crypto_unregister_ahash(&algs[i]); 598 } 599 EXPORT_SYMBOL_GPL(crypto_unregister_ahashes); 600 601 int ahash_register_instance(struct crypto_template *tmpl, 602 struct ahash_instance *inst) 603 { 604 int err; 605 606 if (WARN_ON(!inst->free)) 607 return -EINVAL; 608 609 err = ahash_prepare_alg(&inst->alg); 610 if (err) 611 return err; 612 613 return crypto_register_instance(tmpl, ahash_crypto_instance(inst)); 614 } 615 EXPORT_SYMBOL_GPL(ahash_register_instance); 616 617 bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg) 618 { 619 struct crypto_alg *alg = &halg->base; 620 621 if (alg->cra_type != &crypto_ahash_type) 622 return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg)); 623 624 return __crypto_ahash_alg(alg)->setkey != NULL; 625 } 626 EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey); 627 628 MODULE_LICENSE("GPL"); 629 MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); 630