1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * OpenSSL/Cryptogams accelerated Poly1305 transform for arm64 4 * 5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/hwcap.h> 9 #include <asm/neon.h> 10 #include <asm/simd.h> 11 #include <crypto/internal/poly1305.h> 12 #include <linux/cpufeature.h> 13 #include <linux/jump_label.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/unaligned.h> 17 18 asmlinkage void poly1305_block_init_arch( 19 struct poly1305_block_state *state, 20 const u8 raw_key[POLY1305_BLOCK_SIZE]); 21 EXPORT_SYMBOL_GPL(poly1305_block_init_arch); 22 asmlinkage void poly1305_blocks(struct poly1305_block_state *state, 23 const u8 *src, u32 len, u32 hibit); 24 asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, 25 const u8 *src, u32 len, u32 hibit); 26 asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, 27 u8 digest[POLY1305_DIGEST_SIZE], 28 const u32 nonce[4]); 29 EXPORT_SYMBOL_GPL(poly1305_emit_arch); 30 31 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 32 33 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 34 unsigned int len, u32 padbit) 35 { 36 len = round_down(len, POLY1305_BLOCK_SIZE); 37 if (static_branch_likely(&have_neon) && likely(may_use_simd())) { 38 do { 39 unsigned int todo = min_t(unsigned int, len, SZ_4K); 40 41 kernel_neon_begin(); 42 poly1305_blocks_neon(state, src, todo, padbit); 43 kernel_neon_end(); 44 45 len -= todo; 46 src += todo; 47 } while (len); 48 } else 49 poly1305_blocks(state, src, len, padbit); 50 } 51 EXPORT_SYMBOL_GPL(poly1305_blocks_arch); 52 53 bool poly1305_is_arch_optimized(void) 54 { 55 /* We always can use at least the ARM64 scalar implementation. */ 56 return true; 57 } 58 EXPORT_SYMBOL(poly1305_is_arch_optimized); 59 60 static int __init neon_poly1305_mod_init(void) 61 { 62 if (cpu_have_named_feature(ASIMD)) 63 static_branch_enable(&have_neon); 64 return 0; 65 } 66 subsys_initcall(neon_poly1305_mod_init); 67 68 static void __exit neon_poly1305_mod_exit(void) 69 { 70 } 71 module_exit(neon_poly1305_mod_exit); 72 73 MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)"); 74 MODULE_LICENSE("GPL v2"); 75