1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * arm64-optimized SHA-512 block function
4 *
5 * Copyright 2025 Google LLC
6 */
7
8 #include <asm/neon.h>
9 #include <crypto/internal/simd.h>
10 #include <linux/cpufeature.h>
11
12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns);
13
14 asmlinkage void sha512_block_data_order(struct sha512_block_state *state,
15 const u8 *data, size_t nblocks);
16 asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state,
17 const u8 *data, size_t nblocks);
18
sha512_blocks(struct sha512_block_state * state,const u8 * data,size_t nblocks)19 static void sha512_blocks(struct sha512_block_state *state,
20 const u8 *data, size_t nblocks)
21 {
22 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
23 static_branch_likely(&have_sha512_insns) &&
24 likely(crypto_simd_usable())) {
25 do {
26 size_t rem;
27
28 kernel_neon_begin();
29 rem = __sha512_ce_transform(state, data, nblocks);
30 kernel_neon_end();
31 data += (nblocks - rem) * SHA512_BLOCK_SIZE;
32 nblocks = rem;
33 } while (nblocks);
34 } else {
35 sha512_block_data_order(state, data, nblocks);
36 }
37 }
38
39 #ifdef CONFIG_KERNEL_MODE_NEON
40 #define sha512_mod_init_arch sha512_mod_init_arch
sha512_mod_init_arch(void)41 static inline void sha512_mod_init_arch(void)
42 {
43 if (cpu_have_named_feature(SHA512))
44 static_branch_enable(&have_sha512_insns);
45 }
46 #endif /* CONFIG_KERNEL_MODE_NEON */
47