1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * GHASH and POLYVAL, x86_64 optimized 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/fpu/api.h> 8 #include <linux/cpufeature.h> 9 10 #define NUM_H_POWERS 8 11 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul); 13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); 14 15 asmlinkage void polyval_mul_pclmul(struct polyval_elem *a, 16 const struct polyval_elem *b); 17 asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, 18 const struct polyval_elem *b); 19 20 asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc, 21 const struct polyval_elem *key, 22 const u8 *data, size_t nblocks); 23 asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, 24 const struct polyval_key *key, 25 const u8 *data, size_t nblocks); 26 27 #define polyval_preparekey_arch polyval_preparekey_arch 28 static void polyval_preparekey_arch(struct polyval_key *key, 29 const u8 raw_key[POLYVAL_BLOCK_SIZE]) 30 { 31 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); 32 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); 33 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { 34 kernel_fpu_begin(); 35 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 36 key->h_powers[i] = key->h_powers[i + 1]; 37 polyval_mul_pclmul_avx( 38 &key->h_powers[i], 39 &key->h_powers[NUM_H_POWERS - 1]); 40 } 41 kernel_fpu_end(); 42 } else { 43 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 44 key->h_powers[i] = key->h_powers[i + 1]; 45 polyval_mul_generic(&key->h_powers[i], 46 &key->h_powers[NUM_H_POWERS - 1]); 47 } 48 } 49 } 50 51 static void polyval_mul_x86(struct polyval_elem *a, 52 const struct polyval_elem *b) 53 { 54 if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { 55 kernel_fpu_begin(); 56 if (static_branch_likely(&have_pclmul_avx)) 57 polyval_mul_pclmul_avx(a, b); 58 else 59 polyval_mul_pclmul(a, b); 60 kernel_fpu_end(); 61 } else { 62 polyval_mul_generic(a, b); 63 } 64 } 65 66 #define ghash_mul_arch ghash_mul_arch 67 static void ghash_mul_arch(struct polyval_elem *acc, 68 const struct ghash_key *key) 69 { 70 polyval_mul_x86(acc, &key->h); 71 } 72 73 #define polyval_mul_arch polyval_mul_arch 74 static void polyval_mul_arch(struct polyval_elem *acc, 75 const struct polyval_key *key) 76 { 77 polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]); 78 } 79 80 #define ghash_blocks_arch ghash_blocks_arch 81 static void ghash_blocks_arch(struct polyval_elem *acc, 82 const struct ghash_key *key, 83 const u8 *data, size_t nblocks) 84 { 85 if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { 86 do { 87 /* Allow rescheduling every 4 KiB. */ 88 size_t n = min_t(size_t, nblocks, 89 4096 / GHASH_BLOCK_SIZE); 90 91 kernel_fpu_begin(); 92 ghash_blocks_pclmul(acc, &key->h, data, n); 93 kernel_fpu_end(); 94 data += n * GHASH_BLOCK_SIZE; 95 nblocks -= n; 96 } while (nblocks); 97 } else { 98 ghash_blocks_generic(acc, &key->h, data, nblocks); 99 } 100 } 101 102 #define polyval_blocks_arch polyval_blocks_arch 103 static void polyval_blocks_arch(struct polyval_elem *acc, 104 const struct polyval_key *key, 105 const u8 *data, size_t nblocks) 106 { 107 if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { 108 do { 109 /* Allow rescheduling every 4 KiB. */ 110 size_t n = min_t(size_t, nblocks, 111 4096 / POLYVAL_BLOCK_SIZE); 112 113 kernel_fpu_begin(); 114 polyval_blocks_pclmul_avx(acc, key, data, n); 115 kernel_fpu_end(); 116 data += n * POLYVAL_BLOCK_SIZE; 117 nblocks -= n; 118 } while (nblocks); 119 } else { 120 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], 121 data, nblocks); 122 } 123 } 124 125 #define gf128hash_mod_init_arch gf128hash_mod_init_arch 126 static void gf128hash_mod_init_arch(void) 127 { 128 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 129 static_branch_enable(&have_pclmul); 130 if (boot_cpu_has(X86_FEATURE_AVX)) 131 static_branch_enable(&have_pclmul_avx); 132 } 133 } 134