1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * OpenSSL/Cryptogams accelerated Poly1305 transform for arm64
4 *
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <crypto/internal/poly1305.h>
11 #include <linux/cpufeature.h>
12 #include <linux/jump_label.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/unaligned.h>
16
17 asmlinkage void poly1305_block_init_arch(
18 struct poly1305_block_state *state,
19 const u8 raw_key[POLY1305_BLOCK_SIZE]);
20 EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
21 asmlinkage void poly1305_blocks(struct poly1305_block_state *state,
22 const u8 *src, u32 len, u32 hibit);
23 asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
24 const u8 *src, u32 len, u32 hibit);
25 asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
26 u8 digest[POLY1305_DIGEST_SIZE],
27 const u32 nonce[4]);
28 EXPORT_SYMBOL_GPL(poly1305_emit_arch);
29
30 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
31
poly1305_blocks_arch(struct poly1305_block_state * state,const u8 * src,unsigned int len,u32 padbit)32 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
33 unsigned int len, u32 padbit)
34 {
35 len = round_down(len, POLY1305_BLOCK_SIZE);
36 if (static_branch_likely(&have_neon)) {
37 do {
38 unsigned int todo = min_t(unsigned int, len, SZ_4K);
39
40 kernel_neon_begin();
41 poly1305_blocks_neon(state, src, todo, padbit);
42 kernel_neon_end();
43
44 len -= todo;
45 src += todo;
46 } while (len);
47 } else
48 poly1305_blocks(state, src, len, padbit);
49 }
50 EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
51
poly1305_is_arch_optimized(void)52 bool poly1305_is_arch_optimized(void)
53 {
54 /* We always can use at least the ARM64 scalar implementation. */
55 return true;
56 }
57 EXPORT_SYMBOL(poly1305_is_arch_optimized);
58
neon_poly1305_mod_init(void)59 static int __init neon_poly1305_mod_init(void)
60 {
61 if (cpu_have_named_feature(ASIMD))
62 static_branch_enable(&have_neon);
63 return 0;
64 }
65 subsys_initcall(neon_poly1305_mod_init);
66
neon_poly1305_mod_exit(void)67 static void __exit neon_poly1305_mod_exit(void)
68 {
69 }
70 module_exit(neon_poly1305_mod_exit);
71
72 MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)");
73 MODULE_LICENSE("GPL v2");
74