1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 10 #include <asm/hwcap.h> 11 #include <asm/neon.h> 12 #include <asm/simd.h> 13 14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 16 17 #define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */ 18 19 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 20 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 21 22 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 23 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 24 25 static inline u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len) 26 { 27 if (static_branch_likely(&have_crc32)) 28 return crc32_armv8_le(crc, p, len); 29 return crc32_le_base(crc, p, len); 30 } 31 32 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 33 { 34 if (len >= PMULL_MIN_LEN + 15 && 35 static_branch_likely(&have_pmull) && likely(may_use_simd())) { 36 size_t n = -(uintptr_t)p & 15; 37 38 /* align p to 16-byte boundary */ 39 if (n) { 40 crc = crc32_le_scalar(crc, p, n); 41 p += n; 42 len -= n; 43 } 44 n = round_down(len, 16); 45 kernel_neon_begin(); 46 crc = crc32_pmull_le(p, n, crc); 47 kernel_neon_end(); 48 p += n; 49 len -= n; 50 } 51 return crc32_le_scalar(crc, p, len); 52 } 53 54 static inline u32 crc32c_scalar(u32 crc, const u8 *p, size_t len) 55 { 56 if (static_branch_likely(&have_crc32)) 57 return crc32c_armv8_le(crc, p, len); 58 return crc32c_base(crc, p, len); 59 } 60 61 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 62 { 63 if (len >= PMULL_MIN_LEN + 15 && 64 static_branch_likely(&have_pmull) && likely(may_use_simd())) { 65 size_t n = -(uintptr_t)p & 15; 66 67 /* align p to 16-byte boundary */ 68 if (n) { 69 crc = crc32c_scalar(crc, p, n); 70 p += n; 71 len -= n; 72 } 73 n = round_down(len, 16); 74 kernel_neon_begin(); 75 crc = crc32c_pmull_le(p, n, crc); 76 kernel_neon_end(); 77 p += n; 78 len -= n; 79 } 80 return crc32c_scalar(crc, p, len); 81 } 82 83 #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 84 85 #define crc32_mod_init_arch crc32_mod_init_arch 86 static void crc32_mod_init_arch(void) 87 { 88 if (elf_hwcap2 & HWCAP2_CRC32) 89 static_branch_enable(&have_crc32); 90 if (elf_hwcap2 & HWCAP2_PMULL) 91 static_branch_enable(&have_pmull); 92 } 93 94 static inline u32 crc32_optimizations_arch(void) 95 { 96 if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL)) 97 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 98 return 0; 99 } 100