10f961f9fSEric Biggers // SPDX-License-Identifier: GPL-2.0 20f961f9fSEric Biggers /* 30f961f9fSEric Biggers * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum 40f961f9fSEric Biggers * (AVX2 accelerated version) 50f961f9fSEric Biggers * 60f961f9fSEric Biggers * Copyright 2018 Google LLC 70f961f9fSEric Biggers */ 80f961f9fSEric Biggers 90f961f9fSEric Biggers #include <crypto/internal/hash.h> 10f2abe0d7SEric Biggers #include <crypto/internal/simd.h> 110f961f9fSEric Biggers #include <crypto/nhpoly1305.h> 120f961f9fSEric Biggers #include <linux/module.h> 13*0c3dc787SHerbert Xu #include <linux/sizes.h> 14f2abe0d7SEric Biggers #include <asm/simd.h> 150f961f9fSEric Biggers 160f961f9fSEric Biggers asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, 170f961f9fSEric Biggers u8 hash[NH_HASH_BYTES]); 180f961f9fSEric Biggers 190f961f9fSEric Biggers /* wrapper to avoid indirect call to assembly, which doesn't work with CFI */ 200f961f9fSEric Biggers static void _nh_avx2(const u32 *key, const u8 *message, size_t message_len, 210f961f9fSEric Biggers __le64 hash[NH_NUM_PASSES]) 220f961f9fSEric Biggers { 230f961f9fSEric Biggers nh_avx2(key, message, message_len, (u8 *)hash); 240f961f9fSEric Biggers } 250f961f9fSEric Biggers 260f961f9fSEric Biggers static int nhpoly1305_avx2_update(struct shash_desc *desc, 270f961f9fSEric Biggers const u8 *src, unsigned int srclen) 280f961f9fSEric Biggers { 29f2abe0d7SEric Biggers if (srclen < 64 || !crypto_simd_usable()) 300f961f9fSEric Biggers return crypto_nhpoly1305_update(desc, src, srclen); 310f961f9fSEric Biggers 320f961f9fSEric Biggers do { 33a9a8ba90SJason A. Donenfeld unsigned int n = min_t(unsigned int, srclen, SZ_4K); 340f961f9fSEric Biggers 350f961f9fSEric Biggers kernel_fpu_begin(); 360f961f9fSEric Biggers crypto_nhpoly1305_update_helper(desc, src, n, _nh_avx2); 370f961f9fSEric Biggers kernel_fpu_end(); 380f961f9fSEric Biggers src += n; 390f961f9fSEric Biggers srclen -= n; 400f961f9fSEric Biggers } while (srclen); 410f961f9fSEric Biggers return 0; 420f961f9fSEric Biggers } 430f961f9fSEric Biggers 440f961f9fSEric Biggers static struct shash_alg nhpoly1305_alg = { 450f961f9fSEric Biggers .base.cra_name = "nhpoly1305", 460f961f9fSEric Biggers .base.cra_driver_name = "nhpoly1305-avx2", 470f961f9fSEric Biggers .base.cra_priority = 300, 480f961f9fSEric Biggers .base.cra_ctxsize = sizeof(struct nhpoly1305_key), 490f961f9fSEric Biggers .base.cra_module = THIS_MODULE, 500f961f9fSEric Biggers .digestsize = POLY1305_DIGEST_SIZE, 510f961f9fSEric Biggers .init = crypto_nhpoly1305_init, 520f961f9fSEric Biggers .update = nhpoly1305_avx2_update, 530f961f9fSEric Biggers .final = crypto_nhpoly1305_final, 540f961f9fSEric Biggers .setkey = crypto_nhpoly1305_setkey, 550f961f9fSEric Biggers .descsize = sizeof(struct nhpoly1305_state), 560f961f9fSEric Biggers }; 570f961f9fSEric Biggers 580f961f9fSEric Biggers static int __init nhpoly1305_mod_init(void) 590f961f9fSEric Biggers { 600f961f9fSEric Biggers if (!boot_cpu_has(X86_FEATURE_AVX2) || 610f961f9fSEric Biggers !boot_cpu_has(X86_FEATURE_OSXSAVE)) 620f961f9fSEric Biggers return -ENODEV; 630f961f9fSEric Biggers 640f961f9fSEric Biggers return crypto_register_shash(&nhpoly1305_alg); 650f961f9fSEric Biggers } 660f961f9fSEric Biggers 670f961f9fSEric Biggers static void __exit nhpoly1305_mod_exit(void) 680f961f9fSEric Biggers { 690f961f9fSEric Biggers crypto_unregister_shash(&nhpoly1305_alg); 700f961f9fSEric Biggers } 710f961f9fSEric Biggers 720f961f9fSEric Biggers module_init(nhpoly1305_mod_init); 730f961f9fSEric Biggers module_exit(nhpoly1305_mod_exit); 740f961f9fSEric Biggers 750f961f9fSEric Biggers MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)"); 760f961f9fSEric Biggers MODULE_LICENSE("GPL v2"); 770f961f9fSEric Biggers MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); 780f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305"); 790f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305-avx2"); 80