1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/crc32.h>
4 #include <linux/linkage.h>
5 #include <linux/module.h>
6
7 #include <asm/alternative.h>
8 #include <asm/cpufeature.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11
12 #include <crypto/internal/simd.h>
13
14 // The minimum input length to consider the 4-way interleaved code path
15 static const size_t min_len = 1024;
16
17 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len);
18 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len);
19 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len);
20
21 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
22 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
23 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
24
crc32_le_arch(u32 crc,const u8 * p,size_t len)25 u32 __pure crc32_le_arch(u32 crc, const u8 *p, size_t len)
26 {
27 if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
28 return crc32_le_base(crc, p, len);
29
30 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
31 kernel_neon_begin();
32 crc = crc32_le_arm64_4way(crc, p, len);
33 kernel_neon_end();
34
35 p += round_down(len, 64);
36 len %= 64;
37
38 if (!len)
39 return crc;
40 }
41
42 return crc32_le_arm64(crc, p, len);
43 }
44 EXPORT_SYMBOL(crc32_le_arch);
45
crc32c_le_arch(u32 crc,const u8 * p,size_t len)46 u32 __pure crc32c_le_arch(u32 crc, const u8 *p, size_t len)
47 {
48 if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
49 return crc32c_le_base(crc, p, len);
50
51 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
52 kernel_neon_begin();
53 crc = crc32c_le_arm64_4way(crc, p, len);
54 kernel_neon_end();
55
56 p += round_down(len, 64);
57 len %= 64;
58
59 if (!len)
60 return crc;
61 }
62
63 return crc32c_le_arm64(crc, p, len);
64 }
65 EXPORT_SYMBOL(crc32c_le_arch);
66
crc32_be_arch(u32 crc,const u8 * p,size_t len)67 u32 __pure crc32_be_arch(u32 crc, const u8 *p, size_t len)
68 {
69 if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
70 return crc32_be_base(crc, p, len);
71
72 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
73 kernel_neon_begin();
74 crc = crc32_be_arm64_4way(crc, p, len);
75 kernel_neon_end();
76
77 p += round_down(len, 64);
78 len %= 64;
79
80 if (!len)
81 return crc;
82 }
83
84 return crc32_be_arm64(crc, p, len);
85 }
86 EXPORT_SYMBOL(crc32_be_arch);
87
crc32_optimizations(void)88 u32 crc32_optimizations(void)
89 {
90 if (alternative_has_cap_likely(ARM64_HAS_CRC32))
91 return CRC32_LE_OPTIMIZATION |
92 CRC32_BE_OPTIMIZATION |
93 CRC32C_OPTIMIZATION;
94 return 0;
95 }
96 EXPORT_SYMBOL(crc32_optimizations);
97
98 MODULE_LICENSE("GPL");
99 MODULE_DESCRIPTION("arm64-optimized CRC32 functions");
100