1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * OpenSSL/Cryptogams accelerated Poly1305 transform for ARM
4 *
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <crypto/internal/poly1305.h>
12 #include <linux/cpufeature.h>
13 #include <linux/jump_label.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/unaligned.h>
17
18 asmlinkage void poly1305_block_init_arch(
19 struct poly1305_block_state *state,
20 const u8 raw_key[POLY1305_BLOCK_SIZE]);
21 EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
22 asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state,
23 const u8 *src, u32 len, u32 hibit);
24 asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
25 const u8 *src, u32 len, u32 hibit);
26 asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
27 u8 digest[POLY1305_DIGEST_SIZE],
28 const u32 nonce[4]);
29 EXPORT_SYMBOL_GPL(poly1305_emit_arch);
30
31 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
32
poly1305_blocks_arch(struct poly1305_block_state * state,const u8 * src,unsigned int len,u32 padbit)33 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
34 unsigned int len, u32 padbit)
35 {
36 len = round_down(len, POLY1305_BLOCK_SIZE);
37 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
38 static_branch_likely(&have_neon) && likely(may_use_simd())) {
39 do {
40 unsigned int todo = min_t(unsigned int, len, SZ_4K);
41
42 kernel_neon_begin();
43 poly1305_blocks_neon(state, src, todo, padbit);
44 kernel_neon_end();
45
46 len -= todo;
47 src += todo;
48 } while (len);
49 } else
50 poly1305_blocks_arm(state, src, len, padbit);
51 }
52 EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
53
poly1305_is_arch_optimized(void)54 bool poly1305_is_arch_optimized(void)
55 {
56 /* We always can use at least the ARM scalar implementation. */
57 return true;
58 }
59 EXPORT_SYMBOL(poly1305_is_arch_optimized);
60
arm_poly1305_mod_init(void)61 static int __init arm_poly1305_mod_init(void)
62 {
63 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
64 (elf_hwcap & HWCAP_NEON))
65 static_branch_enable(&have_neon);
66 return 0;
67 }
68 subsys_initcall(arm_poly1305_mod_init);
69
arm_poly1305_mod_exit(void)70 static void __exit arm_poly1305_mod_exit(void)
71 {
72 }
73 module_exit(arm_poly1305_mod_exit);
74
75 MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM");
76 MODULE_LICENSE("GPL v2");
77