1*e3babebfSArd Biesheuvel /* SPDX-License-Identifier: GPL-2.0-only */ 2*e3babebfSArd Biesheuvel /* 3*e3babebfSArd Biesheuvel * CRC64 using ARM PMULL instructions 4*e3babebfSArd Biesheuvel */ 5*e3babebfSArd Biesheuvel 6*e3babebfSArd Biesheuvel #include <asm/simd.h> 7*e3babebfSArd Biesheuvel 8*e3babebfSArd Biesheuvel static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 9*e3babebfSArd Biesheuvel 10*e3babebfSArd Biesheuvel u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); 11*e3babebfSArd Biesheuvel 12*e3babebfSArd Biesheuvel #define crc64_be_arch crc64_be_generic 13*e3babebfSArd Biesheuvel 14*e3babebfSArd Biesheuvel static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 15*e3babebfSArd Biesheuvel { 16*e3babebfSArd Biesheuvel if (len >= 128 && static_branch_likely(&have_pmull) && 17*e3babebfSArd Biesheuvel likely(may_use_simd())) { 18*e3babebfSArd Biesheuvel do { 19*e3babebfSArd Biesheuvel size_t chunk = min_t(size_t, len & ~15, SZ_4K); 20*e3babebfSArd Biesheuvel 21*e3babebfSArd Biesheuvel scoped_ksimd() 22*e3babebfSArd Biesheuvel crc = crc64_nvme_neon(crc, p, chunk); 23*e3babebfSArd Biesheuvel 24*e3babebfSArd Biesheuvel p += chunk; 25*e3babebfSArd Biesheuvel len -= chunk; 26*e3babebfSArd Biesheuvel } while (len >= 128); 27*e3babebfSArd Biesheuvel } 28*e3babebfSArd Biesheuvel return crc64_nvme_generic(crc, p, len); 29*e3babebfSArd Biesheuvel } 30*e3babebfSArd Biesheuvel 31*e3babebfSArd Biesheuvel #define crc64_mod_init_arch crc64_mod_init_arch 32*e3babebfSArd Biesheuvel static void crc64_mod_init_arch(void) 33*e3babebfSArd Biesheuvel { 34*e3babebfSArd Biesheuvel if (elf_hwcap2 & HWCAP2_PMULL) 35*e3babebfSArd Biesheuvel static_branch_enable(&have_pmull); 36*e3babebfSArd Biesheuvel } 37