1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <asm/alternative.h> 4 #include <asm/cpufeature.h> 5 #include <asm/simd.h> 6 7 // The minimum input length to consider the 4-way interleaved code path 8 static const size_t min_len = 1024; 9 10 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 11 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 12 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 13 14 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 15 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 16 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 17 18 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 19 { 20 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 21 return crc32_le_base(crc, p, len); 22 23 if (len >= min_len && cpu_have_named_feature(PMULL) && 24 likely(may_use_simd())) { 25 scoped_ksimd() 26 crc = crc32_le_arm64_4way(crc, p, len); 27 28 p += round_down(len, 64); 29 len %= 64; 30 31 if (!len) 32 return crc; 33 } 34 35 return crc32_le_arm64(crc, p, len); 36 } 37 38 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 39 { 40 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 41 return crc32c_base(crc, p, len); 42 43 if (len >= min_len && cpu_have_named_feature(PMULL) && 44 likely(may_use_simd())) { 45 scoped_ksimd() 46 crc = crc32c_le_arm64_4way(crc, p, len); 47 48 p += round_down(len, 64); 49 len %= 64; 50 51 if (!len) 52 return crc; 53 } 54 55 return crc32c_le_arm64(crc, p, len); 56 } 57 58 static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 59 { 60 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 61 return crc32_be_base(crc, p, len); 62 63 if (len >= min_len && cpu_have_named_feature(PMULL) && 64 likely(may_use_simd())) { 65 scoped_ksimd() 66 crc = crc32_be_arm64_4way(crc, p, len); 67 68 p += round_down(len, 64); 69 len %= 64; 70 71 if (!len) 72 return crc; 73 } 74 75 return crc32_be_arm64(crc, p, len); 76 } 77 78 static inline u32 crc32_optimizations_arch(void) 79 { 80 if (alternative_has_cap_likely(ARM64_HAS_CRC32)) 81 return CRC32_LE_OPTIMIZATION | 82 CRC32_BE_OPTIMIZATION | 83 CRC32C_OPTIMIZATION; 84 return 0; 85 } 86