1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * POLYVAL library functions, x86_64 optimized 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/fpu/api.h> 8 #include <linux/cpufeature.h> 9 10 #define NUM_H_POWERS 8 11 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); 13 14 asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, 15 const struct polyval_elem *b); 16 asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, 17 const struct polyval_key *key, 18 const u8 *data, size_t nblocks); 19 20 static void polyval_preparekey_arch(struct polyval_key *key, 21 const u8 raw_key[POLYVAL_BLOCK_SIZE]) 22 { 23 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); 24 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); 25 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { 26 kernel_fpu_begin(); 27 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 28 key->h_powers[i] = key->h_powers[i + 1]; 29 polyval_mul_pclmul_avx( 30 &key->h_powers[i], 31 &key->h_powers[NUM_H_POWERS - 1]); 32 } 33 kernel_fpu_end(); 34 } else { 35 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 36 key->h_powers[i] = key->h_powers[i + 1]; 37 polyval_mul_generic(&key->h_powers[i], 38 &key->h_powers[NUM_H_POWERS - 1]); 39 } 40 } 41 } 42 43 static void polyval_mul_arch(struct polyval_elem *acc, 44 const struct polyval_key *key) 45 { 46 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { 47 kernel_fpu_begin(); 48 polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); 49 kernel_fpu_end(); 50 } else { 51 polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); 52 } 53 } 54 55 static void polyval_blocks_arch(struct polyval_elem *acc, 56 const struct polyval_key *key, 57 const u8 *data, size_t nblocks) 58 { 59 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { 60 do { 61 /* Allow rescheduling every 4 KiB. */ 62 size_t n = min_t(size_t, nblocks, 63 4096 / POLYVAL_BLOCK_SIZE); 64 65 kernel_fpu_begin(); 66 polyval_blocks_pclmul_avx(acc, key, data, n); 67 kernel_fpu_end(); 68 data += n * POLYVAL_BLOCK_SIZE; 69 nblocks -= n; 70 } while (nblocks); 71 } else { 72 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], 73 data, nblocks); 74 } 75 } 76 77 #define polyval_mod_init_arch polyval_mod_init_arch 78 static void polyval_mod_init_arch(void) 79 { 80 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && 81 boot_cpu_has(X86_FEATURE_AVX)) 82 static_branch_enable(&have_pclmul_avx); 83 } 84