1*b646b782SEric Biggers /* SPDX-License-Identifier: GPL-2.0 */
2*b646b782SEric Biggers /*
3*b646b782SEric Biggers * Poly1305 authenticator algorithm, RFC7539.
4*b646b782SEric Biggers *
5*b646b782SEric Biggers * Copyright 2023- IBM Corp. All rights reserved.
6*b646b782SEric Biggers */
7*b646b782SEric Biggers #include <asm/switch_to.h>
8*b646b782SEric Biggers #include <linux/cpufeature.h>
9*b646b782SEric Biggers #include <linux/jump_label.h>
10*b646b782SEric Biggers #include <linux/kernel.h>
11*b646b782SEric Biggers #include <linux/unaligned.h>
12*b646b782SEric Biggers
13*b646b782SEric Biggers asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
14*b646b782SEric Biggers asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit);
15*b646b782SEric Biggers asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 nonce[4], u8 digest[POLY1305_DIGEST_SIZE]);
16*b646b782SEric Biggers
17*b646b782SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
18*b646b782SEric Biggers
vsx_begin(void)19*b646b782SEric Biggers static void vsx_begin(void)
20*b646b782SEric Biggers {
21*b646b782SEric Biggers preempt_disable();
22*b646b782SEric Biggers enable_kernel_vsx();
23*b646b782SEric Biggers }
24*b646b782SEric Biggers
vsx_end(void)25*b646b782SEric Biggers static void vsx_end(void)
26*b646b782SEric Biggers {
27*b646b782SEric Biggers disable_kernel_vsx();
28*b646b782SEric Biggers preempt_enable();
29*b646b782SEric Biggers }
30*b646b782SEric Biggers
poly1305_block_init(struct poly1305_block_state * dctx,const u8 raw_key[POLY1305_BLOCK_SIZE])31*b646b782SEric Biggers static void poly1305_block_init(struct poly1305_block_state *dctx,
32*b646b782SEric Biggers const u8 raw_key[POLY1305_BLOCK_SIZE])
33*b646b782SEric Biggers {
34*b646b782SEric Biggers if (!static_key_enabled(&have_p10))
35*b646b782SEric Biggers return poly1305_block_init_generic(dctx, raw_key);
36*b646b782SEric Biggers
37*b646b782SEric Biggers dctx->h = (struct poly1305_state){};
38*b646b782SEric Biggers dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0);
39*b646b782SEric Biggers dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8);
40*b646b782SEric Biggers }
41*b646b782SEric Biggers
poly1305_blocks(struct poly1305_block_state * state,const u8 * src,unsigned int len,u32 padbit)42*b646b782SEric Biggers static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
43*b646b782SEric Biggers unsigned int len, u32 padbit)
44*b646b782SEric Biggers {
45*b646b782SEric Biggers if (!static_key_enabled(&have_p10))
46*b646b782SEric Biggers return poly1305_blocks_generic(state, src, len, padbit);
47*b646b782SEric Biggers vsx_begin();
48*b646b782SEric Biggers if (len >= POLY1305_BLOCK_SIZE * 4) {
49*b646b782SEric Biggers poly1305_p10le_4blocks(state, src, len);
50*b646b782SEric Biggers src += len - (len % (POLY1305_BLOCK_SIZE * 4));
51*b646b782SEric Biggers len %= POLY1305_BLOCK_SIZE * 4;
52*b646b782SEric Biggers }
53*b646b782SEric Biggers while (len >= POLY1305_BLOCK_SIZE) {
54*b646b782SEric Biggers poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit);
55*b646b782SEric Biggers len -= POLY1305_BLOCK_SIZE;
56*b646b782SEric Biggers src += POLY1305_BLOCK_SIZE;
57*b646b782SEric Biggers }
58*b646b782SEric Biggers vsx_end();
59*b646b782SEric Biggers }
60*b646b782SEric Biggers
poly1305_emit(const struct poly1305_state * state,u8 digest[POLY1305_DIGEST_SIZE],const u32 nonce[4])61*b646b782SEric Biggers static void poly1305_emit(const struct poly1305_state *state,
62*b646b782SEric Biggers u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4])
63*b646b782SEric Biggers {
64*b646b782SEric Biggers if (!static_key_enabled(&have_p10))
65*b646b782SEric Biggers return poly1305_emit_generic(state, digest, nonce);
66*b646b782SEric Biggers poly1305_emit_64(state, nonce, digest);
67*b646b782SEric Biggers }
68*b646b782SEric Biggers
69*b646b782SEric Biggers #define poly1305_mod_init_arch poly1305_mod_init_arch
poly1305_mod_init_arch(void)70*b646b782SEric Biggers static void poly1305_mod_init_arch(void)
71*b646b782SEric Biggers {
72*b646b782SEric Biggers if (cpu_has_feature(CPU_FTR_ARCH_31))
73*b646b782SEric Biggers static_branch_enable(&have_p10);
74*b646b782SEric Biggers }
75