1012c8238SEric Biggers // SPDX-License-Identifier: GPL-2.0 2012c8238SEric Biggers /* 3012c8238SEric Biggers * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum 4012c8238SEric Biggers * (SSE2 accelerated version) 5012c8238SEric Biggers * 6012c8238SEric Biggers * Copyright 2018 Google LLC 7012c8238SEric Biggers */ 8012c8238SEric Biggers 9012c8238SEric Biggers #include <crypto/internal/hash.h> 10*f2abe0d7SEric Biggers #include <crypto/internal/simd.h> 11012c8238SEric Biggers #include <crypto/nhpoly1305.h> 12012c8238SEric Biggers #include <linux/module.h> 13*f2abe0d7SEric Biggers #include <asm/simd.h> 14012c8238SEric Biggers 15012c8238SEric Biggers asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len, 16012c8238SEric Biggers u8 hash[NH_HASH_BYTES]); 17012c8238SEric Biggers 18012c8238SEric Biggers /* wrapper to avoid indirect call to assembly, which doesn't work with CFI */ 19012c8238SEric Biggers static void _nh_sse2(const u32 *key, const u8 *message, size_t message_len, 20012c8238SEric Biggers __le64 hash[NH_NUM_PASSES]) 21012c8238SEric Biggers { 22012c8238SEric Biggers nh_sse2(key, message, message_len, (u8 *)hash); 23012c8238SEric Biggers } 24012c8238SEric Biggers 25012c8238SEric Biggers static int nhpoly1305_sse2_update(struct shash_desc *desc, 26012c8238SEric Biggers const u8 *src, unsigned int srclen) 27012c8238SEric Biggers { 28*f2abe0d7SEric Biggers if (srclen < 64 || !crypto_simd_usable()) 29012c8238SEric Biggers return crypto_nhpoly1305_update(desc, src, srclen); 30012c8238SEric Biggers 31012c8238SEric Biggers do { 32012c8238SEric Biggers unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE); 33012c8238SEric Biggers 34012c8238SEric Biggers kernel_fpu_begin(); 35012c8238SEric Biggers crypto_nhpoly1305_update_helper(desc, src, n, _nh_sse2); 36012c8238SEric Biggers kernel_fpu_end(); 37012c8238SEric Biggers src += n; 38012c8238SEric Biggers srclen -= n; 39012c8238SEric Biggers } while (srclen); 40012c8238SEric Biggers return 0; 41012c8238SEric Biggers } 42012c8238SEric Biggers 43012c8238SEric Biggers static struct shash_alg nhpoly1305_alg = { 44012c8238SEric Biggers .base.cra_name = "nhpoly1305", 45012c8238SEric Biggers .base.cra_driver_name = "nhpoly1305-sse2", 46012c8238SEric Biggers .base.cra_priority = 200, 47012c8238SEric Biggers .base.cra_ctxsize = sizeof(struct nhpoly1305_key), 48012c8238SEric Biggers .base.cra_module = THIS_MODULE, 49012c8238SEric Biggers .digestsize = POLY1305_DIGEST_SIZE, 50012c8238SEric Biggers .init = crypto_nhpoly1305_init, 51012c8238SEric Biggers .update = nhpoly1305_sse2_update, 52012c8238SEric Biggers .final = crypto_nhpoly1305_final, 53012c8238SEric Biggers .setkey = crypto_nhpoly1305_setkey, 54012c8238SEric Biggers .descsize = sizeof(struct nhpoly1305_state), 55012c8238SEric Biggers }; 56012c8238SEric Biggers 57012c8238SEric Biggers static int __init nhpoly1305_mod_init(void) 58012c8238SEric Biggers { 59012c8238SEric Biggers if (!boot_cpu_has(X86_FEATURE_XMM2)) 60012c8238SEric Biggers return -ENODEV; 61012c8238SEric Biggers 62012c8238SEric Biggers return crypto_register_shash(&nhpoly1305_alg); 63012c8238SEric Biggers } 64012c8238SEric Biggers 65012c8238SEric Biggers static void __exit nhpoly1305_mod_exit(void) 66012c8238SEric Biggers { 67012c8238SEric Biggers crypto_unregister_shash(&nhpoly1305_alg); 68012c8238SEric Biggers } 69012c8238SEric Biggers 70012c8238SEric Biggers module_init(nhpoly1305_mod_init); 71012c8238SEric Biggers module_exit(nhpoly1305_mod_exit); 72012c8238SEric Biggers 73012c8238SEric Biggers MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)"); 74012c8238SEric Biggers MODULE_LICENSE("GPL v2"); 75012c8238SEric Biggers MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); 76012c8238SEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305"); 77012c8238SEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305-sse2"); 78