Lines Matching +full:inline +full:- +full:crypto +full:- +full:engine
1 // SPDX-License-Identifier: GPL-2.0+
12 #include <crypto/engine.h>
13 #include <crypto/hash.h>
14 #include <crypto/internal/hash.h>
15 #include <crypto/sha2.h>
41 static inline int hwh_prepare(struct ahash_request *req, in hwh_prepare()
44 hwh->walkbytes = crypto_hash_walk_first(req, &hwh->walk); in hwh_prepare()
45 if (hwh->walkbytes < 0) in hwh_prepare()
46 return hwh->walkbytes; in hwh_prepare()
47 hwh->walkaddr = hwh->walk.data; in hwh_prepare()
59 static inline int hwh_advance(struct hash_walk_helper *hwh, int n) in hwh_advance()
62 return crypto_hash_walk_done(&hwh->walk, n); in hwh_advance()
64 hwh->walkbytes -= n; in hwh_advance()
65 hwh->walkaddr += n; in hwh_advance()
66 if (hwh->walkbytes > 0) in hwh_advance()
69 hwh->walkbytes = crypto_hash_walk_done(&hwh->walk, 0); in hwh_advance()
70 if (hwh->walkbytes < 0) in hwh_advance()
71 return hwh->walkbytes; in hwh_advance()
73 hwh->walkaddr = hwh->walk.data; in hwh_advance()
80 * blocksize of the used hashing sha2-algorithm function codes. The param block
81 * contains the hash chaining value (cv), the input message bit-length (imbl)
82 * and the hmac-secret (key). To prevent code duplication, the sizes of all
85 * param-block:
86 * +-------+
88 * +-------+
90 * +-------+
92 * +-------+
95 * part | sh2-alg | calculation | size | type
96 * -----+---------+-------------+------+--------
140 /* nr of requests enqueued via crypto engine which use this tfm ctx */
242 return -EINVAL; in hash_key()
255 * make_clrkey_token() - wrap the clear key into a pkey clearkey token.
257 static inline int make_clrkey_token(const u8 *clrkey, size_t clrkeylen, in make_clrkey_token()
264 token->type = 0x00; in make_clrkey_token()
265 token->version = 0x02; in make_clrkey_token()
269 token->keytype = PKEY_KEYTYPE_HMAC_512; in make_clrkey_token()
274 token->keytype = PKEY_KEYTYPE_HMAC_1024; in make_clrkey_token()
278 return -EINVAL; in make_clrkey_token()
280 token->len = blocksize; in make_clrkey_token()
283 rc = hash_key(clrkey, clrkeylen, token->key, digestsize); in make_clrkey_token()
287 memcpy(token->key, clrkey, clrkeylen); in make_clrkey_token()
294 * phmac_tfm_ctx_setkey() - Set key value into tfm context, maybe construct
297 static inline int phmac_tfm_ctx_setkey(struct phmac_tfm_ctx *tfm_ctx, in phmac_tfm_ctx_setkey()
300 if (keylen > sizeof(tfm_ctx->keybuf)) in phmac_tfm_ctx_setkey()
301 return -EINVAL; in phmac_tfm_ctx_setkey()
303 memcpy(tfm_ctx->keybuf, key, keylen); in phmac_tfm_ctx_setkey()
304 tfm_ctx->keylen = keylen; in phmac_tfm_ctx_setkey()
311 * This function may sleep - don't call in non-sleeping context.
313 static inline int convert_key(const u8 *key, unsigned int keylen, in convert_key()
318 pk->len = sizeof(pk->protkey); in convert_key()
322 * of 200, 400, 800 and 1600 ms - in total 3 s. in convert_key()
324 for (rc = -EIO, i = 0; rc && i < 5; i++) { in convert_key()
325 if (rc == -EBUSY && msleep_interruptible((1 << i) * 100)) { in convert_key()
326 rc = -EINTR; in convert_key()
330 pk->protkey, &pk->len, &pk->type, in convert_key()
340 * (Re-)Convert the raw key material from the tfm ctx into a protected
349 * code triggers a (re-)conversion this does no harm. This may lead to
358 spin_lock_bh(&tfm_ctx->pk_lock); in phmac_convert_key()
359 tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; in phmac_convert_key()
360 spin_unlock_bh(&tfm_ctx->pk_lock); in phmac_convert_key()
362 rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk); in phmac_convert_key()
365 spin_lock_bh(&tfm_ctx->pk_lock); in phmac_convert_key()
367 tfm_ctx->pk_state = rc; in phmac_convert_key()
369 tfm_ctx->pk_state = PK_STATE_VALID; in phmac_convert_key()
370 tfm_ctx->pk = pk; in phmac_convert_key()
372 spin_unlock_bh(&tfm_ctx->pk_lock); in phmac_convert_key()
380 * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize
382 static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo, in kmac_sha2_set_imbl()
404 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; in phmac_kmac_update()
405 struct hash_walk_helper *hwh = &req_ctx->hwh; in phmac_kmac_update()
417 while (hwh->walkbytes > 0) { in phmac_kmac_update()
419 offset = ctx->buflen[0] % bs; in phmac_kmac_update()
420 if (offset + hwh->walkbytes < bs) in phmac_kmac_update()
425 n = bs - offset; in phmac_kmac_update()
426 memcpy(ctx->buf + offset, hwh->walkaddr, n); in phmac_kmac_update()
427 ctx->gr0.iimp = 1; in phmac_kmac_update()
429 k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs); in phmac_kmac_update()
439 rc = -EIO; in phmac_kmac_update()
442 /* protected key is invalid and needs re-conversion */ in phmac_kmac_update()
444 rc = -EKEYEXPIRED; in phmac_kmac_update()
450 spin_lock_bh(&tfm_ctx->pk_lock); in phmac_kmac_update()
451 memcpy(ctx->param + SHA2_KEY_OFFSET(bs), in phmac_kmac_update()
452 tfm_ctx->pk.protkey, tfm_ctx->pk.len); in phmac_kmac_update()
453 spin_unlock_bh(&tfm_ctx->pk_lock); in phmac_kmac_update()
455 ctx->buflen[0] += n; in phmac_kmac_update()
456 if (ctx->buflen[0] < n) in phmac_kmac_update()
457 ctx->buflen[1]++; in phmac_kmac_update()
465 while (hwh->walkbytes >= bs) { in phmac_kmac_update()
466 n = (hwh->walkbytes / bs) * bs; in phmac_kmac_update()
467 ctx->gr0.iimp = 1; in phmac_kmac_update()
468 k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, hwh->walkaddr, n); in phmac_kmac_update()
470 ctx->buflen[0] += k; in phmac_kmac_update()
471 if (ctx->buflen[0] < k) in phmac_kmac_update()
472 ctx->buflen[1]++; in phmac_kmac_update()
478 /* protected key is invalid and needs re-conversion */ in phmac_kmac_update()
480 rc = -EKEYEXPIRED; in phmac_kmac_update()
486 spin_lock_bh(&tfm_ctx->pk_lock); in phmac_kmac_update()
487 memcpy(ctx->param + SHA2_KEY_OFFSET(bs), in phmac_kmac_update()
488 tfm_ctx->pk.protkey, tfm_ctx->pk.len); in phmac_kmac_update()
489 spin_unlock_bh(&tfm_ctx->pk_lock); in phmac_kmac_update()
495 if (hwh->walkbytes) { in phmac_kmac_update()
496 memcpy(ctx->buf + offset, hwh->walkaddr, hwh->walkbytes); in phmac_kmac_update()
497 ctx->buflen[0] += hwh->walkbytes; in phmac_kmac_update()
498 if (ctx->buflen[0] < hwh->walkbytes) in phmac_kmac_update()
499 ctx->buflen[1]++; in phmac_kmac_update()
500 rc = hwh_advance(hwh, hwh->walkbytes); in phmac_kmac_update()
505 } /* end of while (hwh->walkbytes > 0) */ in phmac_kmac_update()
517 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; in phmac_kmac_final()
523 n = ctx->buflen[0] % bs; in phmac_kmac_final()
524 ctx->gr0.iimp = 0; in phmac_kmac_final()
525 kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs); in phmac_kmac_final()
527 k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, n); in phmac_kmac_final()
532 rc = -EIO; in phmac_kmac_final()
535 /* protected key is invalid and needs re-conversion */ in phmac_kmac_final()
537 rc = -EKEYEXPIRED; in phmac_kmac_final()
543 spin_lock_bh(&tfm_ctx->pk_lock); in phmac_kmac_final()
544 memcpy(ctx->param + SHA2_KEY_OFFSET(bs), in phmac_kmac_final()
545 tfm_ctx->pk.protkey, tfm_ctx->pk.len); in phmac_kmac_final()
546 spin_unlock_bh(&tfm_ctx->pk_lock); in phmac_kmac_final()
549 memcpy(req->result, ctx->param, ds); in phmac_kmac_final()
561 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; in phmac_init()
572 if (!tfm_ctx->fc) { in phmac_init()
573 rc = -ENOKEY; in phmac_init()
576 kmac_ctx->gr0.fc = tfm_ctx->fc; in phmac_init()
582 spin_lock_bh(&tfm_ctx->pk_lock); in phmac_init()
583 memcpy(kmac_ctx->param + SHA2_KEY_OFFSET(bs), in phmac_init()
584 tfm_ctx->pk.protkey, tfm_ctx->pk.len); in phmac_init()
585 spin_unlock_bh(&tfm_ctx->pk_lock); in phmac_init()
597 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; in phmac_update()
598 struct hash_walk_helper *hwh = &req_ctx->hwh; in phmac_update()
606 /* Try synchronous operation if no active engine usage */ in phmac_update()
607 if (!atomic_read(&tfm_ctx->via_engine_ctr)) { in phmac_update()
615 * requests enqueued via engine, fallback to async. Mark tfm as in phmac_update()
616 * using engine to serialize requests. in phmac_update()
618 if (rc == 0 || rc == -EKEYEXPIRED) { in phmac_update()
619 req_ctx->async_op = OP_UPDATE; in phmac_update()
620 atomic_inc(&tfm_ctx->via_engine_ctr); in phmac_update()
622 if (rc != -EINPROGRESS) in phmac_update()
623 atomic_dec(&tfm_ctx->via_engine_ctr); in phmac_update()
626 if (rc != -EINPROGRESS) { in phmac_update()
641 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; in phmac_final()
644 /* Try synchronous operation if no active engine usage */ in phmac_final()
645 if (!atomic_read(&tfm_ctx->via_engine_ctr)) { in phmac_final()
653 * requests enqueued via engine, fallback to async. Mark tfm as in phmac_final()
654 * using engine to serialize requests. in phmac_final()
656 if (rc == 0 || rc == -EKEYEXPIRED) { in phmac_final()
657 req_ctx->async_op = OP_FINAL; in phmac_final()
658 atomic_inc(&tfm_ctx->via_engine_ctr); in phmac_final()
660 if (rc != -EINPROGRESS) in phmac_final()
661 atomic_dec(&tfm_ctx->via_engine_ctr); in phmac_final()
665 if (rc != -EINPROGRESS) in phmac_final()
676 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; in phmac_finup()
677 struct hash_walk_helper *hwh = &req_ctx->hwh; in phmac_finup()
685 req_ctx->async_op = OP_FINUP; in phmac_finup()
687 /* Try synchronous operations if no active engine usage */ in phmac_finup()
688 if (!atomic_read(&tfm_ctx->via_engine_ctr)) { in phmac_finup()
691 req_ctx->async_op = OP_FINAL; in phmac_finup()
693 if (!rc && req_ctx->async_op == OP_FINAL && in phmac_finup()
694 !atomic_read(&tfm_ctx->via_engine_ctr)) { in phmac_finup()
702 * requests enqueued via engine, fallback to async. Mark tfm as in phmac_finup()
703 * using engine to serialize requests. in phmac_finup()
705 if (rc == 0 || rc == -EKEYEXPIRED) { in phmac_finup()
706 /* req->async_op has been set to either OP_FINUP or OP_FINAL */ in phmac_finup()
707 atomic_inc(&tfm_ctx->via_engine_ctr); in phmac_finup()
709 if (rc != -EINPROGRESS) in phmac_finup()
710 atomic_dec(&tfm_ctx->via_engine_ctr); in phmac_finup()
713 if (rc != -EINPROGRESS) in phmac_finup()
717 if (rc != -EINPROGRESS) in phmac_finup()
757 rc = -ENOMEM; in phmac_setkey()
780 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512) in phmac_setkey()
781 rc = -EINVAL; in phmac_setkey()
783 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_224; in phmac_setkey()
786 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512) in phmac_setkey()
787 rc = -EINVAL; in phmac_setkey()
789 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_256; in phmac_setkey()
792 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024) in phmac_setkey()
793 rc = -EINVAL; in phmac_setkey()
795 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_384; in phmac_setkey()
798 if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024) in phmac_setkey()
799 rc = -EINVAL; in phmac_setkey()
801 tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_512; in phmac_setkey()
804 tfm_ctx->fc = 0; in phmac_setkey()
805 rc = -EINVAL; in phmac_setkey()
817 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; in phmac_export()
827 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; in phmac_import()
840 spin_lock_init(&tfm_ctx->pk_lock); in phmac_init_tfm()
851 memzero_explicit(tfm_ctx->keybuf, sizeof(tfm_ctx->keybuf)); in phmac_exit_tfm()
852 memzero_explicit(&tfm_ctx->pk, sizeof(tfm_ctx->pk)); in phmac_exit_tfm()
855 static int phmac_do_one_request(struct crypto_engine *engine, void *areq) in phmac_do_one_request() argument
861 struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; in phmac_do_one_request()
862 struct hash_walk_helper *hwh = &req_ctx->hwh; in phmac_do_one_request()
863 int rc = -EINVAL; in phmac_do_one_request()
867 * 1. req->async_op == OP_UPDATE with req->nbytes > 0 in phmac_do_one_request()
868 * 2. req->async_op == OP_FINUP with req->nbytes > 0 in phmac_do_one_request()
869 * 3. req->async_op == OP_FINAL in phmac_do_one_request()
874 switch (req_ctx->async_op) { in phmac_do_one_request()
878 if (rc == -EKEYEXPIRED) { in phmac_do_one_request()
881 * Trigger a re-schedule of this request by returning in phmac_do_one_request()
882 * -ENOSPC ("hardware queue full") to the crypto engine. in phmac_do_one_request()
883 * To avoid immediately re-invocation of this callback, in phmac_do_one_request()
888 return -ENOSPC; in phmac_do_one_request()
893 if (req_ctx->async_op == OP_UPDATE) in phmac_do_one_request()
895 req_ctx->async_op = OP_FINAL; in phmac_do_one_request()
899 if (rc == -EKEYEXPIRED) { in phmac_do_one_request()
902 * Trigger a re-schedule of this request by returning in phmac_do_one_request()
903 * -ENOSPC ("hardware queue full") to the crypto engine. in phmac_do_one_request()
904 * To avoid immediately re-invocation of this callback, in phmac_do_one_request()
909 return -ENOSPC; in phmac_do_one_request()
914 return -EOPNOTSUPP; in phmac_do_one_request()
918 if (rc || req_ctx->async_op == OP_FINAL) in phmac_do_one_request()
922 atomic_dec(&tfm_ctx->via_engine_ctr); in phmac_do_one_request()
923 crypto_finalize_hash_request(engine, req, rc); in phmac_do_one_request()
996 for (i = ARRAY_SIZE(phmac_algs) - 1; i >= 0; i--) { in s390_phmac_exit()
998 if (phmac->registered) in s390_phmac_exit()
999 crypto_engine_unregister_ahash(&phmac->alg); in s390_phmac_exit()
1012 return -ENODEV; in s390_phmac_init()
1014 return -ENODEV; in s390_phmac_init()
1021 /* with this pseudo device alloc and start a crypto engine */ in s390_phmac_init()
1026 rc = -ENOMEM; in s390_phmac_init()
1038 if (!cpacf_query_func(CPACF_KMAC, phmac->fc)) in s390_phmac_init()
1040 rc = crypto_engine_register_ahash(&phmac->alg); in s390_phmac_init()
1043 phmac->registered = true; in s390_phmac_init()
1044 pr_debug("%s registered\n", phmac->alg.base.halg.base.cra_name); in s390_phmac_init()