1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * GHASH and POLYVAL, arm64 optimized 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/simd.h> 8 #include <linux/cpufeature.h> 9 10 #define NUM_H_POWERS 8 11 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); 13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 14 15 asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, 16 const u8 *src, 17 const struct polyval_elem *h); 18 asmlinkage void polyval_mul_pmull(struct polyval_elem *a, 19 const struct polyval_elem *b); 20 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, 21 const struct polyval_key *key, 22 const u8 *data, size_t nblocks); 23 24 #define polyval_preparekey_arch polyval_preparekey_arch 25 static void polyval_preparekey_arch(struct polyval_key *key, 26 const u8 raw_key[POLYVAL_BLOCK_SIZE]) 27 { 28 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); 29 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); 30 if (static_branch_likely(&have_pmull) && may_use_simd()) { 31 scoped_ksimd() { 32 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 33 key->h_powers[i] = key->h_powers[i + 1]; 34 polyval_mul_pmull( 35 &key->h_powers[i], 36 &key->h_powers[NUM_H_POWERS - 1]); 37 } 38 } 39 } else { 40 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 41 key->h_powers[i] = key->h_powers[i + 1]; 42 polyval_mul_generic(&key->h_powers[i], 43 &key->h_powers[NUM_H_POWERS - 1]); 44 } 45 } 46 } 47 48 static void polyval_mul_arm64(struct polyval_elem *a, 49 const struct polyval_elem *b) 50 { 51 if (static_branch_likely(&have_asimd) && may_use_simd()) { 52 static const u8 zeroes[GHASH_BLOCK_SIZE]; 53 54 scoped_ksimd() { 55 if (static_branch_likely(&have_pmull)) { 56 polyval_mul_pmull(a, b); 57 } else { 58 /* 59 * Note that this is indeed equivalent to a 60 * POLYVAL multiplication, since it takes the 61 * accumulator and key in POLYVAL format, and 62 * byte-swapping a block of zeroes is a no-op. 63 */ 64 pmull_ghash_update_p8(1, a, zeroes, b); 65 } 66 } 67 } else { 68 polyval_mul_generic(a, b); 69 } 70 } 71 72 #define ghash_mul_arch ghash_mul_arch 73 static void ghash_mul_arch(struct polyval_elem *acc, 74 const struct ghash_key *key) 75 { 76 polyval_mul_arm64(acc, &key->h); 77 } 78 79 #define polyval_mul_arch polyval_mul_arch 80 static void polyval_mul_arch(struct polyval_elem *acc, 81 const struct polyval_key *key) 82 { 83 polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]); 84 } 85 86 #define ghash_blocks_arch ghash_blocks_arch 87 static void ghash_blocks_arch(struct polyval_elem *acc, 88 const struct ghash_key *key, 89 const u8 *data, size_t nblocks) 90 { 91 if (static_branch_likely(&have_asimd) && may_use_simd()) { 92 scoped_ksimd() 93 pmull_ghash_update_p8(nblocks, acc, data, &key->h); 94 } else { 95 ghash_blocks_generic(acc, &key->h, data, nblocks); 96 } 97 } 98 99 #define polyval_blocks_arch polyval_blocks_arch 100 static void polyval_blocks_arch(struct polyval_elem *acc, 101 const struct polyval_key *key, 102 const u8 *data, size_t nblocks) 103 { 104 if (static_branch_likely(&have_pmull) && may_use_simd()) { 105 scoped_ksimd() 106 polyval_blocks_pmull(acc, key, data, nblocks); 107 } else { 108 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], 109 data, nblocks); 110 } 111 } 112 113 #define gf128hash_mod_init_arch gf128hash_mod_init_arch 114 static void gf128hash_mod_init_arch(void) 115 { 116 if (cpu_have_named_feature(ASIMD)) { 117 static_branch_enable(&have_asimd); 118 if (cpu_have_named_feature(PMULL)) 119 static_branch_enable(&have_pmull); 120 } 121 } 122