1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <asm/alternative.h> 4 #include <asm/cpufeature.h> 5 #include <asm/neon.h> 6 #include <asm/simd.h> 7 8 // The minimum input length to consider the 4-way interleaved code path 9 static const size_t min_len = 1024; 10 11 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 12 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 13 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 14 15 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 16 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 17 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 18 19 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 20 { 21 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 22 return crc32_le_base(crc, p, len); 23 24 if (len >= min_len && cpu_have_named_feature(PMULL) && 25 likely(may_use_simd())) { 26 kernel_neon_begin(); 27 crc = crc32_le_arm64_4way(crc, p, len); 28 kernel_neon_end(); 29 30 p += round_down(len, 64); 31 len %= 64; 32 33 if (!len) 34 return crc; 35 } 36 37 return crc32_le_arm64(crc, p, len); 38 } 39 40 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 41 { 42 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 43 return crc32c_base(crc, p, len); 44 45 if (len >= min_len && cpu_have_named_feature(PMULL) && 46 likely(may_use_simd())) { 47 kernel_neon_begin(); 48 crc = crc32c_le_arm64_4way(crc, p, len); 49 kernel_neon_end(); 50 51 p += round_down(len, 64); 52 len %= 64; 53 54 if (!len) 55 return crc; 56 } 57 58 return crc32c_le_arm64(crc, p, len); 59 } 60 61 static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 62 { 63 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 64 return crc32_be_base(crc, p, len); 65 66 if (len >= min_len && cpu_have_named_feature(PMULL) && 67 likely(may_use_simd())) { 68 kernel_neon_begin(); 69 crc = crc32_be_arm64_4way(crc, p, len); 70 kernel_neon_end(); 71 72 p += round_down(len, 64); 73 len %= 64; 74 75 if (!len) 76 return crc; 77 } 78 79 return crc32_be_arm64(crc, p, len); 80 } 81 82 static inline u32 crc32_optimizations_arch(void) 83 { 84 if (alternative_has_cap_likely(ARM64_HAS_CRC32)) 85 return CRC32_LE_OPTIMIZATION | 86 CRC32_BE_OPTIMIZATION | 87 CRC32C_OPTIMIZATION; 88 return 0; 89 } 90