1*60e3f1e9SEric Biggers /* SPDX-License-Identifier: GPL-2.0-or-later */ 2*60e3f1e9SEric Biggers /* 3*60e3f1e9SEric Biggers * arm64-optimized SHA-512 block function 4*60e3f1e9SEric Biggers * 5*60e3f1e9SEric Biggers * Copyright 2025 Google LLC 6*60e3f1e9SEric Biggers */ 7*60e3f1e9SEric Biggers 8*60e3f1e9SEric Biggers #include <asm/neon.h> 9*60e3f1e9SEric Biggers #include <crypto/internal/simd.h> 10*60e3f1e9SEric Biggers #include <linux/cpufeature.h> 11*60e3f1e9SEric Biggers 12*60e3f1e9SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); 13*60e3f1e9SEric Biggers 14*60e3f1e9SEric Biggers asmlinkage void sha512_blocks_arch(struct sha512_block_state *state, 15*60e3f1e9SEric Biggers const u8 *data, size_t nblocks); 16*60e3f1e9SEric Biggers asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, 17*60e3f1e9SEric Biggers const u8 *data, size_t nblocks); 18*60e3f1e9SEric Biggers 19*60e3f1e9SEric Biggers static void sha512_blocks(struct sha512_block_state *state, 20*60e3f1e9SEric Biggers const u8 *data, size_t nblocks) 21*60e3f1e9SEric Biggers { 22*60e3f1e9SEric Biggers if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 23*60e3f1e9SEric Biggers static_branch_likely(&have_sha512_insns) && 24*60e3f1e9SEric Biggers likely(crypto_simd_usable())) { 25*60e3f1e9SEric Biggers do { 26*60e3f1e9SEric Biggers size_t rem; 27*60e3f1e9SEric Biggers 28*60e3f1e9SEric Biggers kernel_neon_begin(); 29*60e3f1e9SEric Biggers rem = __sha512_ce_transform(state, data, nblocks); 30*60e3f1e9SEric Biggers kernel_neon_end(); 31*60e3f1e9SEric Biggers data += (nblocks - rem) * SHA512_BLOCK_SIZE; 32*60e3f1e9SEric Biggers nblocks = rem; 33*60e3f1e9SEric Biggers } while (nblocks); 34*60e3f1e9SEric Biggers } else { 35*60e3f1e9SEric Biggers sha512_blocks_arch(state, data, nblocks); 36*60e3f1e9SEric Biggers } 37*60e3f1e9SEric Biggers } 38*60e3f1e9SEric Biggers 39*60e3f1e9SEric Biggers #ifdef CONFIG_KERNEL_MODE_NEON 40*60e3f1e9SEric Biggers #define sha512_mod_init_arch sha512_mod_init_arch 41*60e3f1e9SEric Biggers static inline void sha512_mod_init_arch(void) 42*60e3f1e9SEric Biggers { 43*60e3f1e9SEric Biggers if (cpu_have_named_feature(SHA512)) 44*60e3f1e9SEric Biggers static_branch_enable(&have_sha512_insns); 45*60e3f1e9SEric Biggers } 46*60e3f1e9SEric Biggers #endif /* CONFIG_KERNEL_MODE_NEON */ 47