161f86c70SEric Biggers // SPDX-License-Identifier: GPL-2.0 261f86c70SEric Biggers /* 361f86c70SEric Biggers * OpenSSL/Cryptogams accelerated Poly1305 transform for arm64 461f86c70SEric Biggers * 561f86c70SEric Biggers * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 661f86c70SEric Biggers */ 761f86c70SEric Biggers 861f86c70SEric Biggers #include <asm/hwcap.h> 961f86c70SEric Biggers #include <asm/neon.h> 10*eec76ea5SEric Biggers #include <asm/simd.h> 1161f86c70SEric Biggers #include <crypto/internal/poly1305.h> 1261f86c70SEric Biggers #include <linux/cpufeature.h> 1361f86c70SEric Biggers #include <linux/jump_label.h> 1461f86c70SEric Biggers #include <linux/kernel.h> 1561f86c70SEric Biggers #include <linux/module.h> 1661f86c70SEric Biggers #include <linux/unaligned.h> 1761f86c70SEric Biggers 1861f86c70SEric Biggers asmlinkage void poly1305_block_init_arch( 1961f86c70SEric Biggers struct poly1305_block_state *state, 2061f86c70SEric Biggers const u8 raw_key[POLY1305_BLOCK_SIZE]); 2161f86c70SEric Biggers EXPORT_SYMBOL_GPL(poly1305_block_init_arch); 2261f86c70SEric Biggers asmlinkage void poly1305_blocks(struct poly1305_block_state *state, 2361f86c70SEric Biggers const u8 *src, u32 len, u32 hibit); 2461f86c70SEric Biggers asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, 2561f86c70SEric Biggers const u8 *src, u32 len, u32 hibit); 2661f86c70SEric Biggers asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, 2761f86c70SEric Biggers u8 digest[POLY1305_DIGEST_SIZE], 2861f86c70SEric Biggers const u32 nonce[4]); 2961f86c70SEric Biggers EXPORT_SYMBOL_GPL(poly1305_emit_arch); 3061f86c70SEric Biggers 3161f86c70SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 3261f86c70SEric Biggers 3361f86c70SEric Biggers void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 3461f86c70SEric Biggers unsigned int len, u32 padbit) 3561f86c70SEric Biggers { 3661f86c70SEric Biggers len = round_down(len, POLY1305_BLOCK_SIZE); 37*eec76ea5SEric Biggers if (static_branch_likely(&have_neon) && likely(may_use_simd())) { 3861f86c70SEric Biggers do { 3961f86c70SEric Biggers unsigned int todo = min_t(unsigned int, len, SZ_4K); 4061f86c70SEric Biggers 4161f86c70SEric Biggers kernel_neon_begin(); 4261f86c70SEric Biggers poly1305_blocks_neon(state, src, todo, padbit); 4361f86c70SEric Biggers kernel_neon_end(); 4461f86c70SEric Biggers 4561f86c70SEric Biggers len -= todo; 4661f86c70SEric Biggers src += todo; 4761f86c70SEric Biggers } while (len); 4861f86c70SEric Biggers } else 4961f86c70SEric Biggers poly1305_blocks(state, src, len, padbit); 5061f86c70SEric Biggers } 5161f86c70SEric Biggers EXPORT_SYMBOL_GPL(poly1305_blocks_arch); 5261f86c70SEric Biggers 5361f86c70SEric Biggers bool poly1305_is_arch_optimized(void) 5461f86c70SEric Biggers { 5561f86c70SEric Biggers /* We always can use at least the ARM64 scalar implementation. */ 5661f86c70SEric Biggers return true; 5761f86c70SEric Biggers } 5861f86c70SEric Biggers EXPORT_SYMBOL(poly1305_is_arch_optimized); 5961f86c70SEric Biggers 6061f86c70SEric Biggers static int __init neon_poly1305_mod_init(void) 6161f86c70SEric Biggers { 6261f86c70SEric Biggers if (cpu_have_named_feature(ASIMD)) 6361f86c70SEric Biggers static_branch_enable(&have_neon); 6461f86c70SEric Biggers return 0; 6561f86c70SEric Biggers } 6661f86c70SEric Biggers subsys_initcall(neon_poly1305_mod_init); 6761f86c70SEric Biggers 6861f86c70SEric Biggers static void __exit neon_poly1305_mod_exit(void) 6961f86c70SEric Biggers { 7061f86c70SEric Biggers } 7161f86c70SEric Biggers module_exit(neon_poly1305_mod_exit); 7261f86c70SEric Biggers 7361f86c70SEric Biggers MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)"); 7461f86c70SEric Biggers MODULE_LICENSE("GPL v2"); 75