1*b646b782SEric Biggers /* SPDX-License-Identifier: GPL-2.0 */
2*b646b782SEric Biggers /*
3*b646b782SEric Biggers * OpenSSL/Cryptogams accelerated Poly1305 transform for ARM
4*b646b782SEric Biggers *
5*b646b782SEric Biggers * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
6*b646b782SEric Biggers */
7*b646b782SEric Biggers
8*b646b782SEric Biggers #include <asm/hwcap.h>
9*b646b782SEric Biggers #include <asm/neon.h>
10*b646b782SEric Biggers #include <asm/simd.h>
11*b646b782SEric Biggers #include <linux/cpufeature.h>
12*b646b782SEric Biggers #include <linux/jump_label.h>
13*b646b782SEric Biggers #include <linux/kernel.h>
14*b646b782SEric Biggers
15*b646b782SEric Biggers asmlinkage void poly1305_block_init(struct poly1305_block_state *state,
16*b646b782SEric Biggers const u8 raw_key[POLY1305_BLOCK_SIZE]);
17*b646b782SEric Biggers asmlinkage void poly1305_blocks_arm(struct poly1305_block_state *state,
18*b646b782SEric Biggers const u8 *src, u32 len, u32 hibit);
19*b646b782SEric Biggers asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
20*b646b782SEric Biggers const u8 *src, u32 len, u32 hibit);
21*b646b782SEric Biggers asmlinkage void poly1305_emit(const struct poly1305_state *state,
22*b646b782SEric Biggers u8 digest[POLY1305_DIGEST_SIZE],
23*b646b782SEric Biggers const u32 nonce[4]);
24*b646b782SEric Biggers
25*b646b782SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
26*b646b782SEric Biggers
poly1305_blocks(struct poly1305_block_state * state,const u8 * src,unsigned int len,u32 padbit)27*b646b782SEric Biggers static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
28*b646b782SEric Biggers unsigned int len, u32 padbit)
29*b646b782SEric Biggers {
30*b646b782SEric Biggers if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
31*b646b782SEric Biggers static_branch_likely(&have_neon) && likely(may_use_simd())) {
32*b646b782SEric Biggers do {
33*b646b782SEric Biggers unsigned int todo = min_t(unsigned int, len, SZ_4K);
34*b646b782SEric Biggers
35*b646b782SEric Biggers kernel_neon_begin();
36*b646b782SEric Biggers poly1305_blocks_neon(state, src, todo, padbit);
37*b646b782SEric Biggers kernel_neon_end();
38*b646b782SEric Biggers
39*b646b782SEric Biggers len -= todo;
40*b646b782SEric Biggers src += todo;
41*b646b782SEric Biggers } while (len);
42*b646b782SEric Biggers } else
43*b646b782SEric Biggers poly1305_blocks_arm(state, src, len, padbit);
44*b646b782SEric Biggers }
45*b646b782SEric Biggers
46*b646b782SEric Biggers #ifdef CONFIG_KERNEL_MODE_NEON
47*b646b782SEric Biggers #define poly1305_mod_init_arch poly1305_mod_init_arch
poly1305_mod_init_arch(void)48*b646b782SEric Biggers static void poly1305_mod_init_arch(void)
49*b646b782SEric Biggers {
50*b646b782SEric Biggers if (elf_hwcap & HWCAP_NEON)
51*b646b782SEric Biggers static_branch_enable(&have_neon);
52*b646b782SEric Biggers }
53*b646b782SEric Biggers #endif /* CONFIG_KERNEL_MODE_NEON */
54