1*e96cb950SEric Biggers /* SPDX-License-Identifier: GPL-2.0-or-later */ 2*e96cb950SEric Biggers /* 3*e96cb950SEric Biggers * SHA-256 optimized for x86_64 4*e96cb950SEric Biggers * 5*e96cb950SEric Biggers * Copyright 2025 Google LLC 6*e96cb950SEric Biggers */ 7*e96cb950SEric Biggers #include <asm/fpu/api.h> 8*e96cb950SEric Biggers #include <crypto/internal/simd.h> 9*e96cb950SEric Biggers #include <linux/static_call.h> 10*e96cb950SEric Biggers 11*e96cb950SEric Biggers asmlinkage void sha256_transform_ssse3(struct sha256_block_state *state, 12*e96cb950SEric Biggers const u8 *data, size_t nblocks); 13*e96cb950SEric Biggers asmlinkage void sha256_transform_avx(struct sha256_block_state *state, 14*e96cb950SEric Biggers const u8 *data, size_t nblocks); 15*e96cb950SEric Biggers asmlinkage void sha256_transform_rorx(struct sha256_block_state *state, 16*e96cb950SEric Biggers const u8 *data, size_t nblocks); 17*e96cb950SEric Biggers asmlinkage void sha256_ni_transform(struct sha256_block_state *state, 18*e96cb950SEric Biggers const u8 *data, size_t nblocks); 19*e96cb950SEric Biggers 20*e96cb950SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); 21*e96cb950SEric Biggers 22*e96cb950SEric Biggers DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); 23*e96cb950SEric Biggers 24*e96cb950SEric Biggers static void sha256_blocks(struct sha256_block_state *state, 25*e96cb950SEric Biggers const u8 *data, size_t nblocks) 26*e96cb950SEric Biggers { 27*e96cb950SEric Biggers if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { 28*e96cb950SEric Biggers kernel_fpu_begin(); 29*e96cb950SEric Biggers static_call(sha256_blocks_x86)(state, data, nblocks); 30*e96cb950SEric Biggers kernel_fpu_end(); 31*e96cb950SEric Biggers } else { 32*e96cb950SEric Biggers sha256_blocks_generic(state, data, nblocks); 33*e96cb950SEric Biggers } 34*e96cb950SEric Biggers } 35*e96cb950SEric Biggers 36*e96cb950SEric Biggers #define sha256_mod_init_arch sha256_mod_init_arch 37*e96cb950SEric Biggers static inline void sha256_mod_init_arch(void) 38*e96cb950SEric Biggers { 39*e96cb950SEric Biggers if (boot_cpu_has(X86_FEATURE_SHA_NI)) { 40*e96cb950SEric Biggers static_call_update(sha256_blocks_x86, sha256_ni_transform); 41*e96cb950SEric Biggers } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 42*e96cb950SEric Biggers NULL) && 43*e96cb950SEric Biggers boot_cpu_has(X86_FEATURE_AVX)) { 44*e96cb950SEric Biggers if (boot_cpu_has(X86_FEATURE_AVX2) && 45*e96cb950SEric Biggers boot_cpu_has(X86_FEATURE_BMI2)) 46*e96cb950SEric Biggers static_call_update(sha256_blocks_x86, 47*e96cb950SEric Biggers sha256_transform_rorx); 48*e96cb950SEric Biggers else 49*e96cb950SEric Biggers static_call_update(sha256_blocks_x86, 50*e96cb950SEric Biggers sha256_transform_avx); 51*e96cb950SEric Biggers } else if (!boot_cpu_has(X86_FEATURE_SSSE3)) { 52*e96cb950SEric Biggers return; 53*e96cb950SEric Biggers } 54*e96cb950SEric Biggers static_branch_enable(&have_sha256_x86); 55*e96cb950SEric Biggers } 56