1*e96cb950SEric Biggers /* SPDX-License-Identifier: GPL-2.0-or-later */
2*e96cb950SEric Biggers /*
3*e96cb950SEric Biggers * SHA-256 optimized for ARM
4*e96cb950SEric Biggers *
5*e96cb950SEric Biggers * Copyright 2025 Google LLC
6*e96cb950SEric Biggers */
7*e96cb950SEric Biggers #include <asm/neon.h>
8*e96cb950SEric Biggers #include <crypto/internal/simd.h>
9*e96cb950SEric Biggers
10*e96cb950SEric Biggers asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
11*e96cb950SEric Biggers const u8 *data, size_t nblocks);
12*e96cb950SEric Biggers asmlinkage void sha256_block_data_order_neon(struct sha256_block_state *state,
13*e96cb950SEric Biggers const u8 *data, size_t nblocks);
14*e96cb950SEric Biggers asmlinkage void sha256_ce_transform(struct sha256_block_state *state,
15*e96cb950SEric Biggers const u8 *data, size_t nblocks);
16*e96cb950SEric Biggers
17*e96cb950SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
18*e96cb950SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
19*e96cb950SEric Biggers
sha256_blocks(struct sha256_block_state * state,const u8 * data,size_t nblocks)20*e96cb950SEric Biggers static void sha256_blocks(struct sha256_block_state *state,
21*e96cb950SEric Biggers const u8 *data, size_t nblocks)
22*e96cb950SEric Biggers {
23*e96cb950SEric Biggers if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
24*e96cb950SEric Biggers static_branch_likely(&have_neon) && crypto_simd_usable()) {
25*e96cb950SEric Biggers kernel_neon_begin();
26*e96cb950SEric Biggers if (static_branch_likely(&have_ce))
27*e96cb950SEric Biggers sha256_ce_transform(state, data, nblocks);
28*e96cb950SEric Biggers else
29*e96cb950SEric Biggers sha256_block_data_order_neon(state, data, nblocks);
30*e96cb950SEric Biggers kernel_neon_end();
31*e96cb950SEric Biggers } else {
32*e96cb950SEric Biggers sha256_block_data_order(state, data, nblocks);
33*e96cb950SEric Biggers }
34*e96cb950SEric Biggers }
35*e96cb950SEric Biggers
36*e96cb950SEric Biggers #ifdef CONFIG_KERNEL_MODE_NEON
37*e96cb950SEric Biggers #define sha256_mod_init_arch sha256_mod_init_arch
sha256_mod_init_arch(void)38*e96cb950SEric Biggers static inline void sha256_mod_init_arch(void)
39*e96cb950SEric Biggers {
40*e96cb950SEric Biggers if (elf_hwcap & HWCAP_NEON) {
41*e96cb950SEric Biggers static_branch_enable(&have_neon);
42*e96cb950SEric Biggers if (elf_hwcap2 & HWCAP2_SHA2)
43*e96cb950SEric Biggers static_branch_enable(&have_ce);
44*e96cb950SEric Biggers }
45*e96cb950SEric Biggers }
46*e96cb950SEric Biggers #endif /* CONFIG_KERNEL_MODE_NEON */
47