1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <asm/alternative.h> 4 #include <asm/cpufeature.h> 5 #include <asm/neon.h> 6 #include <asm/simd.h> 7 8 #include <crypto/internal/simd.h> 9 10 // The minimum input length to consider the 4-way interleaved code path 11 static const size_t min_len = 1024; 12 13 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 14 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 15 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 16 17 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 18 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 19 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 20 21 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 22 { 23 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 24 return crc32_le_base(crc, p, len); 25 26 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 27 kernel_neon_begin(); 28 crc = crc32_le_arm64_4way(crc, p, len); 29 kernel_neon_end(); 30 31 p += round_down(len, 64); 32 len %= 64; 33 34 if (!len) 35 return crc; 36 } 37 38 return crc32_le_arm64(crc, p, len); 39 } 40 41 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 42 { 43 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 44 return crc32c_base(crc, p, len); 45 46 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 47 kernel_neon_begin(); 48 crc = crc32c_le_arm64_4way(crc, p, len); 49 kernel_neon_end(); 50 51 p += round_down(len, 64); 52 len %= 64; 53 54 if (!len) 55 return crc; 56 } 57 58 return crc32c_le_arm64(crc, p, len); 59 } 60 61 static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 62 { 63 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 64 return crc32_be_base(crc, p, len); 65 66 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 67 kernel_neon_begin(); 68 crc = crc32_be_arm64_4way(crc, p, len); 69 kernel_neon_end(); 70 71 p += round_down(len, 64); 72 len %= 64; 73 74 if (!len) 75 return crc; 76 } 77 78 return crc32_be_arm64(crc, p, len); 79 } 80 81 static inline u32 crc32_optimizations_arch(void) 82 { 83 if (alternative_has_cap_likely(ARM64_HAS_CRC32)) 84 return CRC32_LE_OPTIMIZATION | 85 CRC32_BE_OPTIMIZATION | 86 CRC32C_OPTIMIZATION; 87 return 0; 88 } 89