1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * OpenSSL/Cryptogams accelerated Poly1305 transform for ARM 4 * 5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/hwcap.h> 9 #include <asm/neon.h> 10 #include <asm/simd.h> 11 #include <crypto/internal/poly1305.h> 12 #include <linux/cpufeature.h> 13 #include <linux/jump_label.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/unaligned.h> 17 18 asmlinkage void poly1305_block_init_arch( 19 struct poly1305_block_state *state, 20 const u8 raw_key[POLY1305_BLOCK_SIZE]); 21 EXPORT_SYMBOL_GPL(poly1305_block_init_arch); 22 asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state, 23 const u8 *src, u32 len, u32 hibit); 24 asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state, 25 const u8 *src, u32 len, u32 hibit); 26 asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, 27 u8 digest[POLY1305_DIGEST_SIZE], 28 const u32 nonce[4]); 29 EXPORT_SYMBOL_GPL(poly1305_emit_arch); 30 31 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 32 33 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, 34 unsigned int len, u32 padbit) 35 { 36 len = round_down(len, POLY1305_BLOCK_SIZE); 37 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 38 static_branch_likely(&have_neon) && likely(may_use_simd())) { 39 do { 40 unsigned int todo = min_t(unsigned int, len, SZ_4K); 41 42 kernel_neon_begin(); 43 poly1305_blocks_neon(state, src, todo, padbit); 44 kernel_neon_end(); 45 46 len -= todo; 47 src += todo; 48 } while (len); 49 } else 50 poly1305_blocks_arm(state, src, len, padbit); 51 } 52 EXPORT_SYMBOL_GPL(poly1305_blocks_arch); 53 54 bool poly1305_is_arch_optimized(void) 55 { 56 /* We always can use at least the ARM scalar implementation. */ 57 return true; 58 } 59 EXPORT_SYMBOL(poly1305_is_arch_optimized); 60 61 static int __init arm_poly1305_mod_init(void) 62 { 63 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 64 (elf_hwcap & HWCAP_NEON)) 65 static_branch_enable(&have_neon); 66 return 0; 67 } 68 subsys_initcall(arm_poly1305_mod_init); 69 70 static void __exit arm_poly1305_mod_exit(void) 71 { 72 } 73 module_exit(arm_poly1305_mod_exit); 74 75 MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); 76 MODULE_LICENSE("GPL v2"); 77