1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * CRC64 using ARM PMULL instructions 4 */ 5 6 #include <asm/simd.h> 7 8 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 9 10 u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); 11 12 #define crc64_be_arch crc64_be_generic 13 14 static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 15 { 16 if (len >= 128 && static_branch_likely(&have_pmull) && 17 likely(may_use_simd())) { 18 do { 19 size_t chunk = min_t(size_t, len & ~15, SZ_4K); 20 21 scoped_ksimd() 22 crc = crc64_nvme_neon(crc, p, chunk); 23 24 p += chunk; 25 len -= chunk; 26 } while (len >= 128); 27 } 28 return crc64_nvme_generic(crc, p, len); 29 } 30 31 #define crc64_mod_init_arch crc64_mod_init_arch 32 static void crc64_mod_init_arch(void) 33 { 34 if (elf_hwcap2 & HWCAP2_PMULL) 35 static_branch_enable(&have_pmull); 36 } 37