1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * ChaCha stream cipher (P10 accelerated)
4 *
5 * Copyright 2023- IBM Corp. All rights reserved.
6 */
7
8 #include <crypto/chacha.h>
9 #include <crypto/internal/simd.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/cpufeature.h>
13 #include <linux/sizes.h>
14 #include <asm/simd.h>
15 #include <asm/switch_to.h>
16
17 asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst,
18 const u8 *src, unsigned int len, int nrounds);
19
20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
21
vsx_begin(void)22 static void vsx_begin(void)
23 {
24 preempt_disable();
25 enable_kernel_vsx();
26 }
27
vsx_end(void)28 static void vsx_end(void)
29 {
30 disable_kernel_vsx();
31 preempt_enable();
32 }
33
chacha_p10_do_8x(struct chacha_state * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)34 static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src,
35 unsigned int bytes, int nrounds)
36 {
37 unsigned int l = bytes & ~0x0FF;
38
39 if (l > 0) {
40 chacha_p10le_8x(state, dst, src, l, nrounds);
41 bytes -= l;
42 src += l;
43 dst += l;
44 state->x[12] += l / CHACHA_BLOCK_SIZE;
45 }
46
47 if (bytes > 0)
48 chacha_crypt_generic(state, dst, src, bytes, nrounds);
49 }
50
hchacha_block_arch(const struct chacha_state * state,u32 out[HCHACHA_OUT_WORDS],int nrounds)51 void hchacha_block_arch(const struct chacha_state *state,
52 u32 out[HCHACHA_OUT_WORDS], int nrounds)
53 {
54 hchacha_block_generic(state, out, nrounds);
55 }
56 EXPORT_SYMBOL(hchacha_block_arch);
57
chacha_crypt_arch(struct chacha_state * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)58 void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src,
59 unsigned int bytes, int nrounds)
60 {
61 if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
62 !crypto_simd_usable())
63 return chacha_crypt_generic(state, dst, src, bytes, nrounds);
64
65 do {
66 unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
67
68 vsx_begin();
69 chacha_p10_do_8x(state, dst, src, todo, nrounds);
70 vsx_end();
71
72 bytes -= todo;
73 src += todo;
74 dst += todo;
75 } while (bytes);
76 }
77 EXPORT_SYMBOL(chacha_crypt_arch);
78
chacha_is_arch_optimized(void)79 bool chacha_is_arch_optimized(void)
80 {
81 return static_key_enabled(&have_p10);
82 }
83 EXPORT_SYMBOL(chacha_is_arch_optimized);
84
chacha_p10_init(void)85 static int __init chacha_p10_init(void)
86 {
87 if (cpu_has_feature(CPU_FTR_ARCH_31))
88 static_branch_enable(&have_p10);
89 return 0;
90 }
91 subsys_initcall(chacha_p10_init);
92
chacha_p10_exit(void)93 static void __exit chacha_p10_exit(void)
94 {
95 }
96 module_exit(chacha_p10_exit);
97
98 MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)");
99 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
100 MODULE_LICENSE("GPL v2");
101