1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/crc32.h> 4 #include <linux/linkage.h> 5 6 #include <asm/alternative.h> 7 #include <asm/cpufeature.h> 8 #include <asm/neon.h> 9 #include <asm/simd.h> 10 11 #include <crypto/internal/simd.h> 12 13 // The minimum input length to consider the 4-way interleaved code path 14 static const size_t min_len = 1024; 15 16 asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 17 asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 18 asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 19 20 asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 21 asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 22 asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 23 24 u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) 25 { 26 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 27 return crc32_le_base(crc, p, len); 28 29 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 30 kernel_neon_begin(); 31 crc = crc32_le_arm64_4way(crc, p, len); 32 kernel_neon_end(); 33 34 p += round_down(len, 64); 35 len %= 64; 36 37 if (!len) 38 return crc; 39 } 40 41 return crc32_le_arm64(crc, p, len); 42 } 43 44 u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) 45 { 46 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 47 return __crc32c_le_base(crc, p, len); 48 49 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 50 kernel_neon_begin(); 51 crc = crc32c_le_arm64_4way(crc, p, len); 52 kernel_neon_end(); 53 54 p += round_down(len, 64); 55 len %= 64; 56 57 if (!len) 58 return crc; 59 } 60 61 return crc32c_le_arm64(crc, p, len); 62 } 63 64 u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) 65 { 66 if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 67 return crc32_be_base(crc, p, len); 68 69 if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 70 kernel_neon_begin(); 71 crc = crc32_be_arm64_4way(crc, p, len); 72 kernel_neon_end(); 73 74 p += round_down(len, 64); 75 len %= 64; 76 77 if (!len) 78 return crc; 79 } 80 81 return crc32_be_arm64(crc, p, len); 82 } 83