1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * POLYVAL library functions, arm64 optimized 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/simd.h> 8 #include <linux/cpufeature.h> 9 10 #define NUM_H_POWERS 8 11 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 13 14 asmlinkage void polyval_mul_pmull(struct polyval_elem *a, 15 const struct polyval_elem *b); 16 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, 17 const struct polyval_key *key, 18 const u8 *data, size_t nblocks); 19 20 static void polyval_preparekey_arch(struct polyval_key *key, 21 const u8 raw_key[POLYVAL_BLOCK_SIZE]) 22 { 23 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); 24 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); 25 if (static_branch_likely(&have_pmull) && may_use_simd()) { 26 scoped_ksimd() { 27 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 28 key->h_powers[i] = key->h_powers[i + 1]; 29 polyval_mul_pmull( 30 &key->h_powers[i], 31 &key->h_powers[NUM_H_POWERS - 1]); 32 } 33 } 34 } else { 35 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 36 key->h_powers[i] = key->h_powers[i + 1]; 37 polyval_mul_generic(&key->h_powers[i], 38 &key->h_powers[NUM_H_POWERS - 1]); 39 } 40 } 41 } 42 43 static void polyval_mul_arch(struct polyval_elem *acc, 44 const struct polyval_key *key) 45 { 46 if (static_branch_likely(&have_pmull) && may_use_simd()) { 47 scoped_ksimd() 48 polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); 49 } else { 50 polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); 51 } 52 } 53 54 static void polyval_blocks_arch(struct polyval_elem *acc, 55 const struct polyval_key *key, 56 const u8 *data, size_t nblocks) 57 { 58 if (static_branch_likely(&have_pmull) && may_use_simd()) { 59 do { 60 /* Allow rescheduling every 4 KiB. */ 61 size_t n = min_t(size_t, nblocks, 62 4096 / POLYVAL_BLOCK_SIZE); 63 64 scoped_ksimd() 65 polyval_blocks_pmull(acc, key, data, n); 66 data += n * POLYVAL_BLOCK_SIZE; 67 nblocks -= n; 68 } while (nblocks); 69 } else { 70 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], 71 data, nblocks); 72 } 73 } 74 75 #define polyval_mod_init_arch polyval_mod_init_arch 76 static void polyval_mod_init_arch(void) 77 { 78 if (cpu_have_named_feature(PMULL)) 79 static_branch_enable(&have_pmull); 80 } 81