1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * x86_64 accelerated implementation of NH 4 * 5 * Copyright 2018 Google LLC 6 */ 7 8 #include <asm/fpu/api.h> 9 #include <linux/static_call.h> 10 11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2); 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2); 13 14 asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len, 15 __le64 hash[NH_NUM_PASSES]); 16 asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, 17 __le64 hash[NH_NUM_PASSES]); 18 19 static bool nh_arch(const u32 *key, const u8 *message, size_t message_len, 20 __le64 hash[NH_NUM_PASSES]) 21 { 22 if (message_len >= 64 && static_branch_likely(&have_sse2) && 23 irq_fpu_usable()) { 24 kernel_fpu_begin(); 25 if (static_branch_likely(&have_avx2)) 26 nh_avx2(key, message, message_len, hash); 27 else 28 nh_sse2(key, message, message_len, hash); 29 kernel_fpu_end(); 30 return true; 31 } 32 return false; 33 } 34 35 #define nh_mod_init_arch nh_mod_init_arch 36 static void nh_mod_init_arch(void) 37 { 38 if (boot_cpu_has(X86_FEATURE_XMM2)) { 39 static_branch_enable(&have_sse2); 40 if (boot_cpu_has(X86_FEATURE_AVX2) && 41 cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 42 NULL)) 43 static_branch_enable(&have_avx2); 44 } 45 } 46