1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <asm/alternative.h>
4 #include <asm/cpufeature.h>
5 #include <asm/neon.h>
6 #include <asm/simd.h>
7
8 #include <crypto/internal/simd.h>
9
10 // The minimum input length to consider the 4-way interleaved code path
11 static const size_t min_len = 1024;
12
13 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len);
14 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len);
15 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len);
16
17 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
18 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
19 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
20
crc32_le_arch(u32 crc,const u8 * p,size_t len)21 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
22 {
23 if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
24 return crc32_le_base(crc, p, len);
25
26 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
27 kernel_neon_begin();
28 crc = crc32_le_arm64_4way(crc, p, len);
29 kernel_neon_end();
30
31 p += round_down(len, 64);
32 len %= 64;
33
34 if (!len)
35 return crc;
36 }
37
38 return crc32_le_arm64(crc, p, len);
39 }
40
crc32c_arch(u32 crc,const u8 * p,size_t len)41 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
42 {
43 if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
44 return crc32c_base(crc, p, len);
45
46 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
47 kernel_neon_begin();
48 crc = crc32c_le_arm64_4way(crc, p, len);
49 kernel_neon_end();
50
51 p += round_down(len, 64);
52 len %= 64;
53
54 if (!len)
55 return crc;
56 }
57
58 return crc32c_le_arm64(crc, p, len);
59 }
60
crc32_be_arch(u32 crc,const u8 * p,size_t len)61 static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
62 {
63 if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
64 return crc32_be_base(crc, p, len);
65
66 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
67 kernel_neon_begin();
68 crc = crc32_be_arm64_4way(crc, p, len);
69 kernel_neon_end();
70
71 p += round_down(len, 64);
72 len %= 64;
73
74 if (!len)
75 return crc;
76 }
77
78 return crc32_be_arm64(crc, p, len);
79 }
80
crc32_optimizations_arch(void)81 static inline u32 crc32_optimizations_arch(void)
82 {
83 if (alternative_has_cap_likely(ARM64_HAS_CRC32))
84 return CRC32_LE_OPTIMIZATION |
85 CRC32_BE_OPTIMIZATION |
86 CRC32C_OPTIMIZATION;
87 return 0;
88 }
89