1*13cecc52SEric Biggers /* SPDX-License-Identifier: GPL-2.0-or-later */
2*13cecc52SEric Biggers /*
3*13cecc52SEric Biggers * ChaCha stream cipher (P10 accelerated)
4*13cecc52SEric Biggers *
5*13cecc52SEric Biggers * Copyright 2023- IBM Corp. All rights reserved.
6*13cecc52SEric Biggers */
7*13cecc52SEric Biggers
8*13cecc52SEric Biggers #include <crypto/internal/simd.h>
9*13cecc52SEric Biggers #include <linux/kernel.h>
10*13cecc52SEric Biggers #include <linux/cpufeature.h>
11*13cecc52SEric Biggers #include <linux/sizes.h>
12*13cecc52SEric Biggers #include <asm/simd.h>
13*13cecc52SEric Biggers #include <asm/switch_to.h>
14*13cecc52SEric Biggers
15*13cecc52SEric Biggers asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst,
16*13cecc52SEric Biggers const u8 *src, unsigned int len, int nrounds);
17*13cecc52SEric Biggers
18*13cecc52SEric Biggers static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
19*13cecc52SEric Biggers
vsx_begin(void)20*13cecc52SEric Biggers static void vsx_begin(void)
21*13cecc52SEric Biggers {
22*13cecc52SEric Biggers preempt_disable();
23*13cecc52SEric Biggers enable_kernel_vsx();
24*13cecc52SEric Biggers }
25*13cecc52SEric Biggers
vsx_end(void)26*13cecc52SEric Biggers static void vsx_end(void)
27*13cecc52SEric Biggers {
28*13cecc52SEric Biggers disable_kernel_vsx();
29*13cecc52SEric Biggers preempt_enable();
30*13cecc52SEric Biggers }
31*13cecc52SEric Biggers
chacha_p10_do_8x(struct chacha_state * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)32*13cecc52SEric Biggers static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src,
33*13cecc52SEric Biggers unsigned int bytes, int nrounds)
34*13cecc52SEric Biggers {
35*13cecc52SEric Biggers unsigned int l = bytes & ~0x0FF;
36*13cecc52SEric Biggers
37*13cecc52SEric Biggers if (l > 0) {
38*13cecc52SEric Biggers chacha_p10le_8x(state, dst, src, l, nrounds);
39*13cecc52SEric Biggers bytes -= l;
40*13cecc52SEric Biggers src += l;
41*13cecc52SEric Biggers dst += l;
42*13cecc52SEric Biggers state->x[12] += l / CHACHA_BLOCK_SIZE;
43*13cecc52SEric Biggers }
44*13cecc52SEric Biggers
45*13cecc52SEric Biggers if (bytes > 0)
46*13cecc52SEric Biggers chacha_crypt_generic(state, dst, src, bytes, nrounds);
47*13cecc52SEric Biggers }
48*13cecc52SEric Biggers
49*13cecc52SEric Biggers #define hchacha_block_arch hchacha_block_generic /* not implemented yet */
50*13cecc52SEric Biggers
chacha_crypt_arch(struct chacha_state * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)51*13cecc52SEric Biggers static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
52*13cecc52SEric Biggers const u8 *src, unsigned int bytes, int nrounds)
53*13cecc52SEric Biggers {
54*13cecc52SEric Biggers if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
55*13cecc52SEric Biggers !crypto_simd_usable())
56*13cecc52SEric Biggers return chacha_crypt_generic(state, dst, src, bytes, nrounds);
57*13cecc52SEric Biggers
58*13cecc52SEric Biggers do {
59*13cecc52SEric Biggers unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
60*13cecc52SEric Biggers
61*13cecc52SEric Biggers vsx_begin();
62*13cecc52SEric Biggers chacha_p10_do_8x(state, dst, src, todo, nrounds);
63*13cecc52SEric Biggers vsx_end();
64*13cecc52SEric Biggers
65*13cecc52SEric Biggers bytes -= todo;
66*13cecc52SEric Biggers src += todo;
67*13cecc52SEric Biggers dst += todo;
68*13cecc52SEric Biggers } while (bytes);
69*13cecc52SEric Biggers }
70*13cecc52SEric Biggers
71*13cecc52SEric Biggers #define chacha_mod_init_arch chacha_mod_init_arch
chacha_mod_init_arch(void)72*13cecc52SEric Biggers static void chacha_mod_init_arch(void)
73*13cecc52SEric Biggers {
74*13cecc52SEric Biggers if (cpu_has_feature(CPU_FTR_ARCH_31))
75*13cecc52SEric Biggers static_branch_enable(&have_p10);
76*13cecc52SEric Biggers }
77