1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * OpenSSL/Cryptogams accelerated Poly1305 transform for arm64
4 *
5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <crypto/internal/poly1305.h>
12 #include <linux/cpufeature.h>
13 #include <linux/jump_label.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/unaligned.h>
17
18 asmlinkage void poly1305_block_init_arch(
19 struct poly1305_block_state *state,
20 const u8 raw_key[POLY1305_BLOCK_SIZE]);
21 EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
22 asmlinkage void poly1305_blocks(struct poly1305_block_state *state,
23 const u8 *src, u32 len, u32 hibit);
24 asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
25 const u8 *src, u32 len, u32 hibit);
26 asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
27 u8 digest[POLY1305_DIGEST_SIZE],
28 const u32 nonce[4]);
29 EXPORT_SYMBOL_GPL(poly1305_emit_arch);
30
31 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
32
poly1305_blocks_arch(struct poly1305_block_state * state,const u8 * src,unsigned int len,u32 padbit)33 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
34 unsigned int len, u32 padbit)
35 {
36 len = round_down(len, POLY1305_BLOCK_SIZE);
37 if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
38 do {
39 unsigned int todo = min_t(unsigned int, len, SZ_4K);
40
41 kernel_neon_begin();
42 poly1305_blocks_neon(state, src, todo, padbit);
43 kernel_neon_end();
44
45 len -= todo;
46 src += todo;
47 } while (len);
48 } else
49 poly1305_blocks(state, src, len, padbit);
50 }
51 EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
52
poly1305_is_arch_optimized(void)53 bool poly1305_is_arch_optimized(void)
54 {
55 /* We always can use at least the ARM64 scalar implementation. */
56 return true;
57 }
58 EXPORT_SYMBOL(poly1305_is_arch_optimized);
59
neon_poly1305_mod_init(void)60 static int __init neon_poly1305_mod_init(void)
61 {
62 if (cpu_have_named_feature(ASIMD))
63 static_branch_enable(&have_neon);
64 return 0;
65 }
66 subsys_initcall(neon_poly1305_mod_init);
67
neon_poly1305_mod_exit(void)68 static void __exit neon_poly1305_mod_exit(void)
69 {
70 }
71 module_exit(neon_poly1305_mod_exit);
72
73 MODULE_DESCRIPTION("Poly1305 authenticator (ARM64 optimized)");
74 MODULE_LICENSE("GPL v2");
75