1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * POLYVAL library functions, arm64 optimized 4 * 5 * Copyright 2025 Google LLC 6 */ 7 #include <asm/neon.h> 8 #include <asm/simd.h> 9 #include <linux/cpufeature.h> 10 11 #define NUM_H_POWERS 8 12 13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 14 15 asmlinkage void polyval_mul_pmull(struct polyval_elem *a, 16 const struct polyval_elem *b); 17 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, 18 const struct polyval_key *key, 19 const u8 *data, size_t nblocks); 20 21 static void polyval_preparekey_arch(struct polyval_key *key, 22 const u8 raw_key[POLYVAL_BLOCK_SIZE]) 23 { 24 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); 25 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); 26 if (static_branch_likely(&have_pmull) && may_use_simd()) { 27 kernel_neon_begin(); 28 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 29 key->h_powers[i] = key->h_powers[i + 1]; 30 polyval_mul_pmull(&key->h_powers[i], 31 &key->h_powers[NUM_H_POWERS - 1]); 32 } 33 kernel_neon_end(); 34 } else { 35 for (int i = NUM_H_POWERS - 2; i >= 0; i--) { 36 key->h_powers[i] = key->h_powers[i + 1]; 37 polyval_mul_generic(&key->h_powers[i], 38 &key->h_powers[NUM_H_POWERS - 1]); 39 } 40 } 41 } 42 43 static void polyval_mul_arch(struct polyval_elem *acc, 44 const struct polyval_key *key) 45 { 46 if (static_branch_likely(&have_pmull) && may_use_simd()) { 47 kernel_neon_begin(); 48 polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); 49 kernel_neon_end(); 50 } else { 51 polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); 52 } 53 } 54 55 static void polyval_blocks_arch(struct polyval_elem *acc, 56 const struct polyval_key *key, 57 const u8 *data, size_t nblocks) 58 { 59 if (static_branch_likely(&have_pmull) && may_use_simd()) { 60 do { 61 /* Allow rescheduling every 4 KiB. */ 62 size_t n = min_t(size_t, nblocks, 63 4096 / POLYVAL_BLOCK_SIZE); 64 65 kernel_neon_begin(); 66 polyval_blocks_pmull(acc, key, data, n); 67 kernel_neon_end(); 68 data += n * POLYVAL_BLOCK_SIZE; 69 nblocks -= n; 70 } while (nblocks); 71 } else { 72 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], 73 data, nblocks); 74 } 75 } 76 77 #define polyval_mod_init_arch polyval_mod_init_arch 78 static void polyval_mod_init_arch(void) 79 { 80 if (cpu_have_named_feature(PMULL)) 81 static_branch_enable(&have_pmull); 82 } 83