1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * arm64-optimized SHA-512 block function 4 * 5 * Copyright 2025 Google LLC 6 */ 7 8 #include <asm/neon.h> 9 #include <crypto/internal/simd.h> 10 #include <linux/cpufeature.h> 11 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); 13 14 asmlinkage void sha512_block_data_order(struct sha512_block_state *state, 15 const u8 *data, size_t nblocks); 16 asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, 17 const u8 *data, size_t nblocks); 18 19 static void sha512_blocks(struct sha512_block_state *state, 20 const u8 *data, size_t nblocks) 21 { 22 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 23 static_branch_likely(&have_sha512_insns) && 24 likely(crypto_simd_usable())) { 25 do { 26 size_t rem; 27 28 kernel_neon_begin(); 29 rem = __sha512_ce_transform(state, data, nblocks); 30 kernel_neon_end(); 31 data += (nblocks - rem) * SHA512_BLOCK_SIZE; 32 nblocks = rem; 33 } while (nblocks); 34 } else { 35 sha512_block_data_order(state, data, nblocks); 36 } 37 } 38 39 #ifdef CONFIG_KERNEL_MODE_NEON 40 #define sha512_mod_init_arch sha512_mod_init_arch 41 static inline void sha512_mod_init_arch(void) 42 { 43 if (cpu_have_named_feature(SHA512)) 44 static_branch_enable(&have_sha512_insns); 45 } 46 #endif /* CONFIG_KERNEL_MODE_NEON */ 47