11862eb00SEric Biggers // SPDX-License-Identifier: GPL-2.0-or-later
21862eb00SEric Biggers /*
31862eb00SEric Biggers * BLAKE2b digest algorithm, NEON accelerated
41862eb00SEric Biggers *
51862eb00SEric Biggers * Copyright 2020 Google LLC
61862eb00SEric Biggers */
71862eb00SEric Biggers
81862eb00SEric Biggers #include <crypto/internal/blake2b.h>
91862eb00SEric Biggers #include <crypto/internal/hash.h>
101862eb00SEric Biggers #include <crypto/internal/simd.h>
111862eb00SEric Biggers
121862eb00SEric Biggers #include <linux/module.h>
131862eb00SEric Biggers #include <linux/sizes.h>
141862eb00SEric Biggers
151862eb00SEric Biggers #include <asm/neon.h>
161862eb00SEric Biggers #include <asm/simd.h>
171862eb00SEric Biggers
181862eb00SEric Biggers asmlinkage void blake2b_compress_neon(struct blake2b_state *state,
191862eb00SEric Biggers const u8 *block, size_t nblocks, u32 inc);
201862eb00SEric Biggers
blake2b_compress_arch(struct blake2b_state * state,const u8 * block,size_t nblocks,u32 inc)211862eb00SEric Biggers static void blake2b_compress_arch(struct blake2b_state *state,
221862eb00SEric Biggers const u8 *block, size_t nblocks, u32 inc)
231862eb00SEric Biggers {
241862eb00SEric Biggers if (!crypto_simd_usable()) {
251862eb00SEric Biggers blake2b_compress_generic(state, block, nblocks, inc);
261862eb00SEric Biggers return;
271862eb00SEric Biggers }
281862eb00SEric Biggers
291862eb00SEric Biggers do {
301862eb00SEric Biggers const size_t blocks = min_t(size_t, nblocks,
311862eb00SEric Biggers SZ_4K / BLAKE2B_BLOCK_SIZE);
321862eb00SEric Biggers
331862eb00SEric Biggers kernel_neon_begin();
341862eb00SEric Biggers blake2b_compress_neon(state, block, blocks, inc);
351862eb00SEric Biggers kernel_neon_end();
361862eb00SEric Biggers
371862eb00SEric Biggers nblocks -= blocks;
381862eb00SEric Biggers block += blocks * BLAKE2B_BLOCK_SIZE;
391862eb00SEric Biggers } while (nblocks);
401862eb00SEric Biggers }
411862eb00SEric Biggers
crypto_blake2b_update_neon(struct shash_desc * desc,const u8 * in,unsigned int inlen)421862eb00SEric Biggers static int crypto_blake2b_update_neon(struct shash_desc *desc,
431862eb00SEric Biggers const u8 *in, unsigned int inlen)
441862eb00SEric Biggers {
451862eb00SEric Biggers return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch);
461862eb00SEric Biggers }
471862eb00SEric Biggers
crypto_blake2b_final_neon(struct shash_desc * desc,u8 * out)481862eb00SEric Biggers static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out)
491862eb00SEric Biggers {
501862eb00SEric Biggers return crypto_blake2b_final(desc, out, blake2b_compress_arch);
511862eb00SEric Biggers }
521862eb00SEric Biggers
531862eb00SEric Biggers #define BLAKE2B_ALG(name, driver_name, digest_size) \
541862eb00SEric Biggers { \
551862eb00SEric Biggers .base.cra_name = name, \
561862eb00SEric Biggers .base.cra_driver_name = driver_name, \
571862eb00SEric Biggers .base.cra_priority = 200, \
581862eb00SEric Biggers .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \
591862eb00SEric Biggers .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \
601862eb00SEric Biggers .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \
611862eb00SEric Biggers .base.cra_module = THIS_MODULE, \
621862eb00SEric Biggers .digestsize = digest_size, \
631862eb00SEric Biggers .setkey = crypto_blake2b_setkey, \
641862eb00SEric Biggers .init = crypto_blake2b_init, \
651862eb00SEric Biggers .update = crypto_blake2b_update_neon, \
661862eb00SEric Biggers .final = crypto_blake2b_final_neon, \
671862eb00SEric Biggers .descsize = sizeof(struct blake2b_state), \
681862eb00SEric Biggers }
691862eb00SEric Biggers
701862eb00SEric Biggers static struct shash_alg blake2b_neon_algs[] = {
711862eb00SEric Biggers BLAKE2B_ALG("blake2b-160", "blake2b-160-neon", BLAKE2B_160_HASH_SIZE),
721862eb00SEric Biggers BLAKE2B_ALG("blake2b-256", "blake2b-256-neon", BLAKE2B_256_HASH_SIZE),
731862eb00SEric Biggers BLAKE2B_ALG("blake2b-384", "blake2b-384-neon", BLAKE2B_384_HASH_SIZE),
741862eb00SEric Biggers BLAKE2B_ALG("blake2b-512", "blake2b-512-neon", BLAKE2B_512_HASH_SIZE),
751862eb00SEric Biggers };
761862eb00SEric Biggers
blake2b_neon_mod_init(void)771862eb00SEric Biggers static int __init blake2b_neon_mod_init(void)
781862eb00SEric Biggers {
791862eb00SEric Biggers if (!(elf_hwcap & HWCAP_NEON))
801862eb00SEric Biggers return -ENODEV;
811862eb00SEric Biggers
821862eb00SEric Biggers return crypto_register_shashes(blake2b_neon_algs,
831862eb00SEric Biggers ARRAY_SIZE(blake2b_neon_algs));
841862eb00SEric Biggers }
851862eb00SEric Biggers
blake2b_neon_mod_exit(void)861862eb00SEric Biggers static void __exit blake2b_neon_mod_exit(void)
871862eb00SEric Biggers {
88*6131e970SEric Biggers crypto_unregister_shashes(blake2b_neon_algs,
891862eb00SEric Biggers ARRAY_SIZE(blake2b_neon_algs));
901862eb00SEric Biggers }
911862eb00SEric Biggers
921862eb00SEric Biggers module_init(blake2b_neon_mod_init);
931862eb00SEric Biggers module_exit(blake2b_neon_mod_exit);
941862eb00SEric Biggers
951862eb00SEric Biggers MODULE_DESCRIPTION("BLAKE2b digest algorithm, NEON accelerated");
961862eb00SEric Biggers MODULE_LICENSE("GPL");
971862eb00SEric Biggers MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
981862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-160");
991862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-160-neon");
1001862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-256");
1011862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-256-neon");
1021862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-384");
1031862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-384-neon");
1041862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-512");
1051862eb00SEric Biggers MODULE_ALIAS_CRYPTO("blake2b-512-neon");
106