1*061cef5fSArd Biesheuvel // SPDX-License-Identifier: GPL-2.0-only 2*061cef5fSArd Biesheuvel /* 3*061cef5fSArd Biesheuvel * Accelerated CRC64 (NVMe) using ARM NEON C intrinsics 4*061cef5fSArd Biesheuvel */ 5*061cef5fSArd Biesheuvel 6*061cef5fSArd Biesheuvel #include <linux/types.h> 7*061cef5fSArd Biesheuvel #include <asm/neon-intrinsics.h> 8*061cef5fSArd Biesheuvel 9*061cef5fSArd Biesheuvel #include "crc64-neon.h" 10*061cef5fSArd Biesheuvel 11*061cef5fSArd Biesheuvel u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); 12*061cef5fSArd Biesheuvel 13*061cef5fSArd Biesheuvel /* x^191 mod G, x^127 mod G */ 14*061cef5fSArd Biesheuvel static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, 15*061cef5fSArd Biesheuvel 0x21e9761e252621acULL }; 16*061cef5fSArd Biesheuvel /* floor(x^127 / G), (G - x^64) / x */ 17*061cef5fSArd Biesheuvel static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL, 18*061cef5fSArd Biesheuvel 0x34d926535897936aULL }; 19*061cef5fSArd Biesheuvel 20*061cef5fSArd Biesheuvel u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len) 21*061cef5fSArd Biesheuvel { 22*061cef5fSArd Biesheuvel uint64x2_t fold_consts = vld1q_u64(fold_consts_val); 23*061cef5fSArd Biesheuvel uint64x2_t v0 = { crc, 0 }; 24*061cef5fSArd Biesheuvel uint64x2_t zero = { }; 25*061cef5fSArd Biesheuvel 26*061cef5fSArd Biesheuvel for (;;) { 27*061cef5fSArd Biesheuvel v0 ^= vreinterpretq_u64_u8(vld1q_u8(p)); 28*061cef5fSArd Biesheuvel 29*061cef5fSArd Biesheuvel p += 16; 30*061cef5fSArd Biesheuvel len -= 16; 31*061cef5fSArd Biesheuvel if (len < 16) 32*061cef5fSArd Biesheuvel break; 33*061cef5fSArd Biesheuvel 34*061cef5fSArd Biesheuvel v0 = pmull64(fold_consts, v0) ^ pmull64_high(fold_consts, v0); 35*061cef5fSArd Biesheuvel } 36*061cef5fSArd Biesheuvel 37*061cef5fSArd Biesheuvel /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */ 38*061cef5fSArd Biesheuvel v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0); 39*061cef5fSArd Biesheuvel 40*061cef5fSArd Biesheuvel /* Final Barrett reduction */ 41*061cef5fSArd Biesheuvel uint64x2_t bconsts = vld1q_u64(bconsts_val); 42*061cef5fSArd Biesheuvel uint64x2_t final = pmull64(bconsts, v0); 43*061cef5fSArd Biesheuvel 44*061cef5fSArd Biesheuvel v0 ^= vextq_u64(zero, final, 1) ^ pmull64_hi_lo(bconsts, final); 45*061cef5fSArd Biesheuvel 46*061cef5fSArd Biesheuvel return vgetq_lane_u64(v0, 1); 47*061cef5fSArd Biesheuvel } 48