1*4a32e5dcSEric Biggers // SPDX-License-Identifier: GPL-2.0 2*4a32e5dcSEric Biggers /* 3*4a32e5dcSEric Biggers * OpenSSL/Cryptogams accelerated Poly1305 transform for ARM 4*4a32e5dcSEric Biggers * 5*4a32e5dcSEric Biggers * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 6*4a32e5dcSEric Biggers */ 7*4a32e5dcSEric Biggers 8*4a32e5dcSEric Biggers #include <asm/hwcap.h> 9*4a32e5dcSEric Biggers #include <asm/neon.h> 10*4a32e5dcSEric Biggers #include <crypto/internal/poly1305.h> 11*4a32e5dcSEric Biggers #include <linux/cpufeature.h> 12*4a32e5dcSEric Biggers #include <linux/jump_label.h> 13*4a32e5dcSEric Biggers #include <linux/kernel.h> 14*4a32e5dcSEric Biggers #include <linux/module.h> 15*4a32e5dcSEric Biggers #include <linux/unaligned.h> 16*4a32e5dcSEric Biggers 17*4a32e5dcSEric Biggers asmlinkage void poly1305_block_init_arch( 18*4a32e5dcSEric Biggers struct poly1305_block_state *state, 19*4a32e5dcSEric Biggers const u8 raw_key[POLY1305_BLOCK_SIZE]); 20*4a32e5dcSEric Biggers EXPORT_SYMBOL_GPL(poly1305_block_init_arch); 21*4a32e5dcSEric Biggers asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state, 22*4a32e5dcSEric Biggers const u8 *src, u32 len, u32 hibit); 23*4a32e5dcSEric Biggers asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, 24*4a32e5dcSEric Biggers const u8 *src, u32 len, u32 hibit); 25*4a32e5dcSEric Biggers asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, 26*4a32e5dcSEric Biggers u8 digest[POLY1305_DIGEST_SIZE], 27*4a32e5dcSEric Biggers const u32 nonce[4]); 28*4a32e5dcSEric Biggers EXPORT_SYMBOL_GPL(poly1305_emit_arch); 29*4a32e5dcSEric Biggers 30*4a32e5dcSEric Biggers void __weak poly1305_blocks_neon(struct poly1305_block_state *state, 31*4a32e5dcSEric Biggers const u8 *src, u32 len, u32 hibit) 32*4a32e5dcSEric Biggers { 33*4a32e5dcSEric Biggers } 34*4a32e5dcSEric Biggers 35*4a32e5dcSEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 36*4a32e5dcSEric Biggers 37*4a32e5dcSEric Biggers void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 38*4a32e5dcSEric Biggers unsigned int len, u32 padbit) 39*4a32e5dcSEric Biggers { 40*4a32e5dcSEric Biggers len = round_down(len, POLY1305_BLOCK_SIZE); 41*4a32e5dcSEric Biggers if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 42*4a32e5dcSEric Biggers static_branch_likely(&have_neon)) { 43*4a32e5dcSEric Biggers do { 44*4a32e5dcSEric Biggers unsigned int todo = min_t(unsigned int, len, SZ_4K); 45*4a32e5dcSEric Biggers 46*4a32e5dcSEric Biggers kernel_neon_begin(); 47*4a32e5dcSEric Biggers poly1305_blocks_neon(state, src, todo, padbit); 48*4a32e5dcSEric Biggers kernel_neon_end(); 49*4a32e5dcSEric Biggers 50*4a32e5dcSEric Biggers len -= todo; 51*4a32e5dcSEric Biggers src += todo; 52*4a32e5dcSEric Biggers } while (len); 53*4a32e5dcSEric Biggers } else 54*4a32e5dcSEric Biggers poly1305_blocks_arm(state, src, len, padbit); 55*4a32e5dcSEric Biggers } 56*4a32e5dcSEric Biggers EXPORT_SYMBOL_GPL(poly1305_blocks_arch); 57*4a32e5dcSEric Biggers 58*4a32e5dcSEric Biggers bool poly1305_is_arch_optimized(void) 59*4a32e5dcSEric Biggers { 60*4a32e5dcSEric Biggers /* We always can use at least the ARM scalar implementation. */ 61*4a32e5dcSEric Biggers return true; 62*4a32e5dcSEric Biggers } 63*4a32e5dcSEric Biggers EXPORT_SYMBOL(poly1305_is_arch_optimized); 64*4a32e5dcSEric Biggers 65*4a32e5dcSEric Biggers static int __init arm_poly1305_mod_init(void) 66*4a32e5dcSEric Biggers { 67*4a32e5dcSEric Biggers if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 68*4a32e5dcSEric Biggers (elf_hwcap & HWCAP_NEON)) 69*4a32e5dcSEric Biggers static_branch_enable(&have_neon); 70*4a32e5dcSEric Biggers return 0; 71*4a32e5dcSEric Biggers } 72*4a32e5dcSEric Biggers subsys_initcall(arm_poly1305_mod_init); 73*4a32e5dcSEric Biggers 74*4a32e5dcSEric Biggers static void __exit arm_poly1305_mod_exit(void) 75*4a32e5dcSEric Biggers { 76*4a32e5dcSEric Biggers } 77*4a32e5dcSEric Biggers module_exit(arm_poly1305_mod_exit); 78*4a32e5dcSEric Biggers 79*4a32e5dcSEric Biggers MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); 80*4a32e5dcSEric Biggers MODULE_LICENSE("GPL v2"); 81