1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * x86-optimized SHA-512 block function 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/fpu/api.h> 8 #include <linux/static_call.h> 9 10 DEFINE_STATIC_CALL(sha512_blocks_x86, sha512_blocks_generic); 11 12 #define DEFINE_X86_SHA512_FN(c_fn, asm_fn) \ 13 asmlinkage void asm_fn(struct sha512_block_state *state, \ 14 const u8 *data, size_t nblocks); \ 15 static void c_fn(struct sha512_block_state *state, const u8 *data, \ 16 size_t nblocks) \ 17 { \ 18 if (likely(irq_fpu_usable())) { \ 19 kernel_fpu_begin(); \ 20 asm_fn(state, data, nblocks); \ 21 kernel_fpu_end(); \ 22 } else { \ 23 sha512_blocks_generic(state, data, nblocks); \ 24 } \ 25 } 26 27 DEFINE_X86_SHA512_FN(sha512_blocks_ssse3, sha512_transform_ssse3); 28 DEFINE_X86_SHA512_FN(sha512_blocks_avx, sha512_transform_avx); 29 DEFINE_X86_SHA512_FN(sha512_blocks_avx2, sha512_transform_rorx); 30 31 static void sha512_blocks(struct sha512_block_state *state, 32 const u8 *data, size_t nblocks) 33 { 34 static_call(sha512_blocks_x86)(state, data, nblocks); 35 } 36 37 #define sha512_mod_init_arch sha512_mod_init_arch 38 static void sha512_mod_init_arch(void) 39 { 40 if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && 41 boot_cpu_has(X86_FEATURE_AVX)) { 42 if (boot_cpu_has(X86_FEATURE_AVX2) && 43 boot_cpu_has(X86_FEATURE_BMI2)) 44 static_call_update(sha512_blocks_x86, 45 sha512_blocks_avx2); 46 else 47 static_call_update(sha512_blocks_x86, 48 sha512_blocks_avx); 49 } else if (boot_cpu_has(X86_FEATURE_SSSE3)) { 50 static_call_update(sha512_blocks_x86, sha512_blocks_ssse3); 51 } 52 } 53