1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include <asm/cpufeature.h> 7 #include <asm/fpu/api.h> 8 #include <asm/processor.h> 9 #include <asm/simd.h> 10 #include <linux/jump_label.h> 11 #include <linux/kernel.h> 12 #include <linux/sizes.h> 13 14 asmlinkage void blake2s_compress_ssse3(struct blake2s_ctx *ctx, 15 const u8 *data, size_t nblocks, u32 inc); 16 asmlinkage void blake2s_compress_avx512(struct blake2s_ctx *ctx, 17 const u8 *data, size_t nblocks, u32 inc); 18 19 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3); 20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512); 21 22 static void blake2s_compress(struct blake2s_ctx *ctx, 23 const u8 *data, size_t nblocks, u32 inc) 24 { 25 /* SIMD disables preemption, so relax after processing each page. */ 26 BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8); 27 28 if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) { 29 blake2s_compress_generic(ctx, data, nblocks, inc); 30 return; 31 } 32 33 do { 34 const size_t blocks = min_t(size_t, nblocks, 35 SZ_4K / BLAKE2S_BLOCK_SIZE); 36 37 kernel_fpu_begin(); 38 if (static_branch_likely(&blake2s_use_avx512)) 39 blake2s_compress_avx512(ctx, data, blocks, inc); 40 else 41 blake2s_compress_ssse3(ctx, data, blocks, inc); 42 kernel_fpu_end(); 43 44 data += blocks * BLAKE2S_BLOCK_SIZE; 45 nblocks -= blocks; 46 } while (nblocks); 47 } 48 49 #define blake2s_mod_init_arch blake2s_mod_init_arch 50 static void blake2s_mod_init_arch(void) 51 { 52 if (boot_cpu_has(X86_FEATURE_SSSE3)) 53 static_branch_enable(&blake2s_use_ssse3); 54 55 if (boot_cpu_has(X86_FEATURE_AVX) && 56 boot_cpu_has(X86_FEATURE_AVX2) && 57 boot_cpu_has(X86_FEATURE_AVX512F) && 58 boot_cpu_has(X86_FEATURE_AVX512VL) && 59 cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | 60 XFEATURE_MASK_AVX512, NULL)) 61 static_branch_enable(&blake2s_use_avx512); 62 } 63