1*372ff60aSEric Biggers // SPDX-License-Identifier: GPL-2.0-only
2*372ff60aSEric Biggers #include <linux/crc32.h>
3*372ff60aSEric Biggers #include <crypto/internal/simd.h>
4*372ff60aSEric Biggers #include <linux/init.h>
5*372ff60aSEric Biggers #include <linux/module.h>
6*372ff60aSEric Biggers #include <linux/kernel.h>
7*372ff60aSEric Biggers #include <linux/cpufeature.h>
8*372ff60aSEric Biggers #include <asm/simd.h>
9*372ff60aSEric Biggers #include <asm/switch_to.h>
10*372ff60aSEric Biggers
11*372ff60aSEric Biggers #define VMX_ALIGN 16
12*372ff60aSEric Biggers #define VMX_ALIGN_MASK (VMX_ALIGN-1)
13*372ff60aSEric Biggers
14*372ff60aSEric Biggers #define VECTOR_BREAKPOINT 512
15*372ff60aSEric Biggers
16*372ff60aSEric Biggers static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
17*372ff60aSEric Biggers
18*372ff60aSEric Biggers u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len);
19*372ff60aSEric Biggers
crc32_le_arch(u32 crc,const u8 * p,size_t len)20*372ff60aSEric Biggers u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
21*372ff60aSEric Biggers {
22*372ff60aSEric Biggers return crc32_le_base(crc, p, len);
23*372ff60aSEric Biggers }
24*372ff60aSEric Biggers EXPORT_SYMBOL(crc32_le_arch);
25*372ff60aSEric Biggers
crc32c_le_arch(u32 crc,const u8 * p,size_t len)26*372ff60aSEric Biggers u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
27*372ff60aSEric Biggers {
28*372ff60aSEric Biggers unsigned int prealign;
29*372ff60aSEric Biggers unsigned int tail;
30*372ff60aSEric Biggers
31*372ff60aSEric Biggers if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32*372ff60aSEric Biggers !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33*372ff60aSEric Biggers return crc32c_le_base(crc, p, len);
34*372ff60aSEric Biggers
35*372ff60aSEric Biggers if ((unsigned long)p & VMX_ALIGN_MASK) {
36*372ff60aSEric Biggers prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
37*372ff60aSEric Biggers crc = crc32c_le_base(crc, p, prealign);
38*372ff60aSEric Biggers len -= prealign;
39*372ff60aSEric Biggers p += prealign;
40*372ff60aSEric Biggers }
41*372ff60aSEric Biggers
42*372ff60aSEric Biggers if (len & ~VMX_ALIGN_MASK) {
43*372ff60aSEric Biggers preempt_disable();
44*372ff60aSEric Biggers pagefault_disable();
45*372ff60aSEric Biggers enable_kernel_altivec();
46*372ff60aSEric Biggers crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
47*372ff60aSEric Biggers disable_kernel_altivec();
48*372ff60aSEric Biggers pagefault_enable();
49*372ff60aSEric Biggers preempt_enable();
50*372ff60aSEric Biggers }
51*372ff60aSEric Biggers
52*372ff60aSEric Biggers tail = len & VMX_ALIGN_MASK;
53*372ff60aSEric Biggers if (tail) {
54*372ff60aSEric Biggers p += len & ~VMX_ALIGN_MASK;
55*372ff60aSEric Biggers crc = crc32c_le_base(crc, p, tail);
56*372ff60aSEric Biggers }
57*372ff60aSEric Biggers
58*372ff60aSEric Biggers return crc;
59*372ff60aSEric Biggers }
60*372ff60aSEric Biggers EXPORT_SYMBOL(crc32c_le_arch);
61*372ff60aSEric Biggers
crc32_be_arch(u32 crc,const u8 * p,size_t len)62*372ff60aSEric Biggers u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
63*372ff60aSEric Biggers {
64*372ff60aSEric Biggers return crc32_be_base(crc, p, len);
65*372ff60aSEric Biggers }
66*372ff60aSEric Biggers EXPORT_SYMBOL(crc32_be_arch);
67*372ff60aSEric Biggers
crc32_powerpc_init(void)68*372ff60aSEric Biggers static int __init crc32_powerpc_init(void)
69*372ff60aSEric Biggers {
70*372ff60aSEric Biggers if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
71*372ff60aSEric Biggers (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
72*372ff60aSEric Biggers static_branch_enable(&have_vec_crypto);
73*372ff60aSEric Biggers return 0;
74*372ff60aSEric Biggers }
75*372ff60aSEric Biggers arch_initcall(crc32_powerpc_init);
76*372ff60aSEric Biggers
crc32_powerpc_exit(void)77*372ff60aSEric Biggers static void __exit crc32_powerpc_exit(void)
78*372ff60aSEric Biggers {
79*372ff60aSEric Biggers }
80*372ff60aSEric Biggers module_exit(crc32_powerpc_exit);
81*372ff60aSEric Biggers
crc32_optimizations(void)82*372ff60aSEric Biggers u32 crc32_optimizations(void)
83*372ff60aSEric Biggers {
84*372ff60aSEric Biggers if (static_key_enabled(&have_vec_crypto))
85*372ff60aSEric Biggers return CRC32C_OPTIMIZATION;
86*372ff60aSEric Biggers return 0;
87*372ff60aSEric Biggers }
88*372ff60aSEric Biggers EXPORT_SYMBOL(crc32_optimizations);
89*372ff60aSEric Biggers
90*372ff60aSEric Biggers MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
91*372ff60aSEric Biggers MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
92*372ff60aSEric Biggers MODULE_LICENSE("GPL");
93