160e3f1e9SEric Biggers /* SPDX-License-Identifier: GPL-2.0-or-later */ 260e3f1e9SEric Biggers /* 360e3f1e9SEric Biggers * arm64-optimized SHA-512 block function 460e3f1e9SEric Biggers * 560e3f1e9SEric Biggers * Copyright 2025 Google LLC 660e3f1e9SEric Biggers */ 760e3f1e9SEric Biggers 860e3f1e9SEric Biggers #include <asm/neon.h> 960e3f1e9SEric Biggers #include <crypto/internal/simd.h> 1060e3f1e9SEric Biggers #include <linux/cpufeature.h> 1160e3f1e9SEric Biggers 1260e3f1e9SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); 1360e3f1e9SEric Biggers 14*9f97707bSEric Biggers asmlinkage void sha512_block_data_order(struct sha512_block_state *state, 1560e3f1e9SEric Biggers const u8 *data, size_t nblocks); 1660e3f1e9SEric Biggers asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, 1760e3f1e9SEric Biggers const u8 *data, size_t nblocks); 1860e3f1e9SEric Biggers 1960e3f1e9SEric Biggers static void sha512_blocks(struct sha512_block_state *state, 2060e3f1e9SEric Biggers const u8 *data, size_t nblocks) 2160e3f1e9SEric Biggers { 2260e3f1e9SEric Biggers if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 2360e3f1e9SEric Biggers static_branch_likely(&have_sha512_insns) && 2460e3f1e9SEric Biggers likely(crypto_simd_usable())) { 2560e3f1e9SEric Biggers do { 2660e3f1e9SEric Biggers size_t rem; 2760e3f1e9SEric Biggers 2860e3f1e9SEric Biggers kernel_neon_begin(); 2960e3f1e9SEric Biggers rem = __sha512_ce_transform(state, data, nblocks); 3060e3f1e9SEric Biggers kernel_neon_end(); 3160e3f1e9SEric Biggers data += (nblocks - rem) * SHA512_BLOCK_SIZE; 3260e3f1e9SEric Biggers nblocks = rem; 3360e3f1e9SEric Biggers } while (nblocks); 3460e3f1e9SEric Biggers } else { 35*9f97707bSEric Biggers sha512_block_data_order(state, data, nblocks); 3660e3f1e9SEric Biggers } 3760e3f1e9SEric Biggers } 3860e3f1e9SEric Biggers 3960e3f1e9SEric Biggers #ifdef CONFIG_KERNEL_MODE_NEON 4060e3f1e9SEric Biggers #define sha512_mod_init_arch sha512_mod_init_arch 4160e3f1e9SEric Biggers static inline void sha512_mod_init_arch(void) 4260e3f1e9SEric Biggers { 4360e3f1e9SEric Biggers if (cpu_have_named_feature(SHA512)) 4460e3f1e9SEric Biggers static_branch_enable(&have_sha512_insns); 4560e3f1e9SEric Biggers } 4660e3f1e9SEric Biggers #endif /* CONFIG_KERNEL_MODE_NEON */ 47