10f961f9fSEric Biggers // SPDX-License-Identifier: GPL-2.0
20f961f9fSEric Biggers /*
30f961f9fSEric Biggers * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
40f961f9fSEric Biggers * (AVX2 accelerated version)
50f961f9fSEric Biggers *
60f961f9fSEric Biggers * Copyright 2018 Google LLC
70f961f9fSEric Biggers */
80f961f9fSEric Biggers
90f961f9fSEric Biggers #include <crypto/internal/hash.h>
10f2abe0d7SEric Biggers #include <crypto/internal/simd.h>
110f961f9fSEric Biggers #include <crypto/nhpoly1305.h>
120f961f9fSEric Biggers #include <linux/module.h>
130c3dc787SHerbert Xu #include <linux/sizes.h>
14f2abe0d7SEric Biggers #include <asm/simd.h>
150f961f9fSEric Biggers
160f961f9fSEric Biggers asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
170f8bc4bdSEric Biggers __le64 hash[NH_NUM_PASSES]);
180f961f9fSEric Biggers
nhpoly1305_avx2_update(struct shash_desc * desc,const u8 * src,unsigned int srclen)190f961f9fSEric Biggers static int nhpoly1305_avx2_update(struct shash_desc *desc,
200f961f9fSEric Biggers const u8 *src, unsigned int srclen)
210f961f9fSEric Biggers {
22f2abe0d7SEric Biggers if (srclen < 64 || !crypto_simd_usable())
230f961f9fSEric Biggers return crypto_nhpoly1305_update(desc, src, srclen);
240f961f9fSEric Biggers
250f961f9fSEric Biggers do {
26a9a8ba90SJason A. Donenfeld unsigned int n = min_t(unsigned int, srclen, SZ_4K);
270f961f9fSEric Biggers
280f961f9fSEric Biggers kernel_fpu_begin();
290f8bc4bdSEric Biggers crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);
300f961f9fSEric Biggers kernel_fpu_end();
310f961f9fSEric Biggers src += n;
320f961f9fSEric Biggers srclen -= n;
330f961f9fSEric Biggers } while (srclen);
340f961f9fSEric Biggers return 0;
350f961f9fSEric Biggers }
360f961f9fSEric Biggers
nhpoly1305_avx2_digest(struct shash_desc * desc,const u8 * src,unsigned int srclen,u8 * out)37*796b06f5SEric Biggers static int nhpoly1305_avx2_digest(struct shash_desc *desc,
38*796b06f5SEric Biggers const u8 *src, unsigned int srclen, u8 *out)
39*796b06f5SEric Biggers {
40*796b06f5SEric Biggers return crypto_nhpoly1305_init(desc) ?:
41*796b06f5SEric Biggers nhpoly1305_avx2_update(desc, src, srclen) ?:
42*796b06f5SEric Biggers crypto_nhpoly1305_final(desc, out);
43*796b06f5SEric Biggers }
44*796b06f5SEric Biggers
450f961f9fSEric Biggers static struct shash_alg nhpoly1305_alg = {
460f961f9fSEric Biggers .base.cra_name = "nhpoly1305",
470f961f9fSEric Biggers .base.cra_driver_name = "nhpoly1305-avx2",
480f961f9fSEric Biggers .base.cra_priority = 300,
490f961f9fSEric Biggers .base.cra_ctxsize = sizeof(struct nhpoly1305_key),
500f961f9fSEric Biggers .base.cra_module = THIS_MODULE,
510f961f9fSEric Biggers .digestsize = POLY1305_DIGEST_SIZE,
520f961f9fSEric Biggers .init = crypto_nhpoly1305_init,
530f961f9fSEric Biggers .update = nhpoly1305_avx2_update,
540f961f9fSEric Biggers .final = crypto_nhpoly1305_final,
55*796b06f5SEric Biggers .digest = nhpoly1305_avx2_digest,
560f961f9fSEric Biggers .setkey = crypto_nhpoly1305_setkey,
570f961f9fSEric Biggers .descsize = sizeof(struct nhpoly1305_state),
580f961f9fSEric Biggers };
590f961f9fSEric Biggers
nhpoly1305_mod_init(void)600f961f9fSEric Biggers static int __init nhpoly1305_mod_init(void)
610f961f9fSEric Biggers {
620f961f9fSEric Biggers if (!boot_cpu_has(X86_FEATURE_AVX2) ||
630f961f9fSEric Biggers !boot_cpu_has(X86_FEATURE_OSXSAVE))
640f961f9fSEric Biggers return -ENODEV;
650f961f9fSEric Biggers
660f961f9fSEric Biggers return crypto_register_shash(&nhpoly1305_alg);
670f961f9fSEric Biggers }
680f961f9fSEric Biggers
nhpoly1305_mod_exit(void)690f961f9fSEric Biggers static void __exit nhpoly1305_mod_exit(void)
700f961f9fSEric Biggers {
710f961f9fSEric Biggers crypto_unregister_shash(&nhpoly1305_alg);
720f961f9fSEric Biggers }
730f961f9fSEric Biggers
740f961f9fSEric Biggers module_init(nhpoly1305_mod_init);
750f961f9fSEric Biggers module_exit(nhpoly1305_mod_exit);
760f961f9fSEric Biggers
770f961f9fSEric Biggers MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
780f961f9fSEric Biggers MODULE_LICENSE("GPL v2");
790f961f9fSEric Biggers MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
800f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305");
810f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
82