163432fd6SDemian Shulhan // SPDX-License-Identifier: GPL-2.0-only 263432fd6SDemian Shulhan /* 363432fd6SDemian Shulhan * Accelerated CRC64 (NVMe) using ARM NEON C intrinsics 463432fd6SDemian Shulhan */ 563432fd6SDemian Shulhan 663432fd6SDemian Shulhan #include <linux/types.h> 763432fd6SDemian Shulhan #include <asm/neon-intrinsics.h> 863432fd6SDemian Shulhan 963432fd6SDemian Shulhan u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); 1063432fd6SDemian Shulhan 1163432fd6SDemian Shulhan /* x^191 mod G, x^127 mod G */ 1263432fd6SDemian Shulhan static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, 1363432fd6SDemian Shulhan 0x21e9761e252621acULL }; 1463432fd6SDemian Shulhan /* floor(x^127 / G), (G - x^64) / x */ 1563432fd6SDemian Shulhan static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL, 1663432fd6SDemian Shulhan 0x34d926535897936aULL }; 1763432fd6SDemian Shulhan 18*8fdef85dSArd Biesheuvel static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) 19*8fdef85dSArd Biesheuvel { 20*8fdef85dSArd Biesheuvel return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), 21*8fdef85dSArd Biesheuvel vgetq_lane_u64(b, 0))); 22*8fdef85dSArd Biesheuvel } 23*8fdef85dSArd Biesheuvel 24*8fdef85dSArd Biesheuvel static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) 25*8fdef85dSArd Biesheuvel { 26*8fdef85dSArd Biesheuvel poly64x2_t l = vreinterpretq_p64_u64(a); 27*8fdef85dSArd Biesheuvel poly64x2_t m = vreinterpretq_p64_u64(b); 28*8fdef85dSArd Biesheuvel 29*8fdef85dSArd Biesheuvel return vreinterpretq_u64_p128(vmull_high_p64(l, m)); 30*8fdef85dSArd Biesheuvel } 31*8fdef85dSArd Biesheuvel 32*8fdef85dSArd Biesheuvel static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) 33*8fdef85dSArd Biesheuvel { 34*8fdef85dSArd Biesheuvel return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), 35*8fdef85dSArd Biesheuvel vgetq_lane_u64(b, 0))); 36*8fdef85dSArd Biesheuvel } 37*8fdef85dSArd Biesheuvel 3863432fd6SDemian Shulhan u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len) 3963432fd6SDemian Shulhan { 40*8fdef85dSArd Biesheuvel uint64x2_t fold_consts = vld1q_u64(fold_consts_val); 41*8fdef85dSArd Biesheuvel uint64x2_t v0 = { crc, 0 }; 42*8fdef85dSArd Biesheuvel uint64x2_t zero = { }; 4363432fd6SDemian Shulhan 44*8fdef85dSArd Biesheuvel for (;;) { 45*8fdef85dSArd Biesheuvel v0 ^= vreinterpretq_u64_u8(vld1q_u8(p)); 4663432fd6SDemian Shulhan 4763432fd6SDemian Shulhan p += 16; 4863432fd6SDemian Shulhan len -= 16; 49*8fdef85dSArd Biesheuvel if (len < 16) 50*8fdef85dSArd Biesheuvel break; 51*8fdef85dSArd Biesheuvel 52*8fdef85dSArd Biesheuvel v0 = pmull64(fold_consts, v0) ^ pmull64_high(fold_consts, v0); 53*8fdef85dSArd Biesheuvel } 5463432fd6SDemian Shulhan 5563432fd6SDemian Shulhan /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */ 56*8fdef85dSArd Biesheuvel v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0); 5763432fd6SDemian Shulhan 5863432fd6SDemian Shulhan /* Final Barrett reduction */ 59*8fdef85dSArd Biesheuvel uint64x2_t bconsts = vld1q_u64(bconsts_val); 60*8fdef85dSArd Biesheuvel uint64x2_t final = pmull64(bconsts, v0); 6163432fd6SDemian Shulhan 62*8fdef85dSArd Biesheuvel v0 ^= vextq_u64(zero, final, 1) ^ pmull64_hi_lo(bconsts, final); 6363432fd6SDemian Shulhan 64*8fdef85dSArd Biesheuvel return vgetq_lane_u64(v0, 1); 6563432fd6SDemian Shulhan } 66