1*61f86c70SEric Biggers // SPDX-License-Identifier: GPL-2.0 2*61f86c70SEric Biggers /* 3*61f86c70SEric Biggers * OpenSSL/Cryptogams accelerated Poly1305 transform for arm64 4*61f86c70SEric Biggers * 5*61f86c70SEric Biggers * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 6*61f86c70SEric Biggers */ 7*61f86c70SEric Biggers 8*61f86c70SEric Biggers #include <asm/hwcap.h> 9*61f86c70SEric Biggers #include <asm/neon.h> 10*61f86c70SEric Biggers #include <crypto/internal/poly1305.h> 11*61f86c70SEric Biggers #include <linux/cpufeature.h> 12*61f86c70SEric Biggers #include <linux/jump_label.h> 13*61f86c70SEric Biggers #include <linux/kernel.h> 14*61f86c70SEric Biggers #include <linux/module.h> 15*61f86c70SEric Biggers #include <linux/unaligned.h> 16*61f86c70SEric Biggers 17*61f86c70SEric Biggers asmlinkage void poly1305_block_init_arch( 18*61f86c70SEric Biggers struct poly1305_block_state *state, 19*61f86c70SEric Biggers const u8 raw_key[POLY1305_BLOCK_SIZE]); 20*61f86c70SEric Biggers EXPORT_SYMBOL_GPL(poly1305_block_init_arch); 21*61f86c70SEric Biggers asmlinkage void poly1305_blocks(struct poly1305_block_state *state, 22*61f86c70SEric Biggers const u8 *src, u32 len, u32 hibit); 23*61f86c70SEric Biggers asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, 24*61f86c70SEric Biggers const u8 *src, u32 len, u32 hibit); 25*61f86c70SEric Biggers asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, 26*61f86c70SEric Biggers u8 digest[POLY1305_DIGEST_SIZE], 27*61f86c70SEric Biggers const u32 nonce[4]); 28*61f86c70SEric Biggers EXPORT_SYMBOL_GPL(poly1305_emit_arch); 29*61f86c70SEric Biggers 30*61f86c70SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 31*61f86c70SEric Biggers 32*61f86c70SEric Biggers void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 33*61f86c70SEric Biggers unsigned int len, u32 padbit) 34*61f86c70SEric Biggers { 35*61f86c70SEric Biggers len = round_down(len, POLY1305_BLOCK_SIZE); 36*61f86c70SEric Biggers if (static_branch_likely(&have_neon)) { 37*61f86c70SEric Biggers do { 38*61f86c70SEric Biggers unsigned int todo = min_t(unsigned int, len, SZ_4K); 39*61f86c70SEric Biggers 40*61f86c70SEric Biggers kernel_neon_begin(); 41*61f86c70SEric Biggers poly1305_blocks_neon(state, src, todo, padbit); 42*61f86c70SEric Biggers kernel_neon_end(); 43*61f86c70SEric Biggers 44*61f86c70SEric Biggers len -= todo; 45*61f86c70SEric Biggers src += todo; 46*61f86c70SEric Biggers } while (len); 47*61f86c70SEric Biggers } else 48*61f86c70SEric Biggers poly1305_blocks(state, src, len, padbit); 49*61f86c70SEric Biggers } 50*61f86c70SEric Biggers EXPORT_SYMBOL_GPL(poly1305_blocks_arch); 51*61f86c70SEric Biggers 52*61f86c70SEric Biggers bool poly1305_is_arch_optimized(void) 53*61f86c70SEric Biggers { 54*61f86c70SEric Biggers /* We always can use at least the ARM64 scalar implementation. */ 55*61f86c70SEric Biggers return true; 56*61f86c70SEric Biggers } 57*61f86c70SEric Biggers EXPORT_SYMBOL(poly1305_is_arch_optimized); 58*61f86c70SEric Biggers 59*61f86c70SEric Biggers static int __init neon_poly1305_mod_init(void) 60*61f86c70SEric Biggers { 61*61f86c70SEric Biggers if (cpu_have_named_feature(ASIMD)) 62*61f86c70SEric Biggers static_branch_enable(&have_neon); 63*61f86c70SEric Biggers return 0; 64*61f86c70SEric Biggers } 65*61f86c70SEric Biggers subsys_initcall(neon_poly1305_mod_init); 66*61f86c70SEric Biggers 67*61f86c70SEric Biggers static void __exit neon_poly1305_mod_exit(void) 68*61f86c70SEric Biggers { 69*61f86c70SEric Biggers } 70*61f86c70SEric Biggers module_exit(neon_poly1305_mod_exit); 71*61f86c70SEric Biggers 72*61f86c70SEric Biggers MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)"); 73*61f86c70SEric Biggers MODULE_LICENSE("GPL v2"); 74