1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * CRC64 using [V]PCLMULQDQ instructions 4 * 5 * Copyright 2025 Google LLC 6 */ 7 8 #include "crc-pclmul-template.h" 9 10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 11 12 DECLARE_CRC_PCLMUL_FUNCS(crc64_msb, u64); 13 DECLARE_CRC_PCLMUL_FUNCS(crc64_lsb, u64); 14 15 static inline u64 crc64_be_arch(u64 crc, const u8 *p, size_t len) 16 { 17 CRC_PCLMUL(crc, p, len, crc64_msb, crc64_msb_0x42f0e1eba9ea3693_consts, 18 have_pclmulqdq); 19 return crc64_be_generic(crc, p, len); 20 } 21 22 static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 23 { 24 CRC_PCLMUL(crc, p, len, crc64_lsb, crc64_lsb_0x9a6c9329ac4bc9b5_consts, 25 have_pclmulqdq); 26 return crc64_nvme_generic(crc, p, len); 27 } 28 29 #define crc64_mod_init_arch crc64_mod_init_arch 30 static inline void crc64_mod_init_arch(void) 31 { 32 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 33 static_branch_enable(&have_pclmulqdq); 34 if (have_vpclmul()) { 35 if (have_avx512()) { 36 static_call_update(crc64_msb_pclmul, 37 crc64_msb_vpclmul_avx512); 38 static_call_update(crc64_lsb_pclmul, 39 crc64_lsb_vpclmul_avx512); 40 } else { 41 static_call_update(crc64_msb_pclmul, 42 crc64_msb_vpclmul_avx2); 43 static_call_update(crc64_lsb_pclmul, 44 crc64_lsb_vpclmul_avx2); 45 } 46 } 47 } 48 } 49