1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * POLYVAL library functions, arm64 optimized
4 *
5 * Copyright 2025 Google LLC
6 */
7 #include <asm/simd.h>
8 #include <linux/cpufeature.h>
9
10 #define NUM_H_POWERS 8
11
12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
13
14 asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
15 const struct polyval_elem *b);
16 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
17 const struct polyval_key *key,
18 const u8 *data, size_t nblocks);
19
polyval_preparekey_arch(struct polyval_key * key,const u8 raw_key[POLYVAL_BLOCK_SIZE])20 static void polyval_preparekey_arch(struct polyval_key *key,
21 const u8 raw_key[POLYVAL_BLOCK_SIZE])
22 {
23 static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
24 memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
25 if (static_branch_likely(&have_pmull) && may_use_simd()) {
26 scoped_ksimd() {
27 for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
28 key->h_powers[i] = key->h_powers[i + 1];
29 polyval_mul_pmull(
30 &key->h_powers[i],
31 &key->h_powers[NUM_H_POWERS - 1]);
32 }
33 }
34 } else {
35 for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
36 key->h_powers[i] = key->h_powers[i + 1];
37 polyval_mul_generic(&key->h_powers[i],
38 &key->h_powers[NUM_H_POWERS - 1]);
39 }
40 }
41 }
42
polyval_mul_arch(struct polyval_elem * acc,const struct polyval_key * key)43 static void polyval_mul_arch(struct polyval_elem *acc,
44 const struct polyval_key *key)
45 {
46 if (static_branch_likely(&have_pmull) && may_use_simd()) {
47 scoped_ksimd()
48 polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
49 } else {
50 polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
51 }
52 }
53
polyval_blocks_arch(struct polyval_elem * acc,const struct polyval_key * key,const u8 * data,size_t nblocks)54 static void polyval_blocks_arch(struct polyval_elem *acc,
55 const struct polyval_key *key,
56 const u8 *data, size_t nblocks)
57 {
58 if (static_branch_likely(&have_pmull) && may_use_simd()) {
59 do {
60 /* Allow rescheduling every 4 KiB. */
61 size_t n = min_t(size_t, nblocks,
62 4096 / POLYVAL_BLOCK_SIZE);
63
64 scoped_ksimd()
65 polyval_blocks_pmull(acc, key, data, n);
66 data += n * POLYVAL_BLOCK_SIZE;
67 nblocks -= n;
68 } while (nblocks);
69 } else {
70 polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
71 data, nblocks);
72 }
73 }
74
75 #define polyval_mod_init_arch polyval_mod_init_arch
polyval_mod_init_arch(void)76 static void polyval_mod_init_arch(void)
77 {
78 if (cpu_have_named_feature(PMULL))
79 static_branch_enable(&have_pmull);
80 }
81