1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * CRC64 using [V]PCLMULQDQ instructions
4 *
5 * Copyright 2025 Google LLC
6 */
7
8 #include "crc-pclmul-template.h"
9
10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
11
12 DECLARE_CRC_PCLMUL_FUNCS(crc64_msb, u64);
13 DECLARE_CRC_PCLMUL_FUNCS(crc64_lsb, u64);
14
crc64_be_arch(u64 crc,const u8 * p,size_t len)15 static inline u64 crc64_be_arch(u64 crc, const u8 *p, size_t len)
16 {
17 CRC_PCLMUL(crc, p, len, crc64_msb, crc64_msb_0x42f0e1eba9ea3693_consts,
18 have_pclmulqdq);
19 return crc64_be_generic(crc, p, len);
20 }
21
crc64_nvme_arch(u64 crc,const u8 * p,size_t len)22 static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
23 {
24 CRC_PCLMUL(crc, p, len, crc64_lsb, crc64_lsb_0x9a6c9329ac4bc9b5_consts,
25 have_pclmulqdq);
26 return crc64_nvme_generic(crc, p, len);
27 }
28
29 #define crc64_mod_init_arch crc64_mod_init_arch
crc64_mod_init_arch(void)30 static inline void crc64_mod_init_arch(void)
31 {
32 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
33 static_branch_enable(&have_pclmulqdq);
34 if (have_vpclmul()) {
35 if (have_avx512()) {
36 static_call_update(crc64_msb_pclmul,
37 crc64_msb_vpclmul_avx512);
38 static_call_update(crc64_lsb_pclmul,
39 crc64_lsb_vpclmul_avx512);
40 } else {
41 static_call_update(crc64_msb_pclmul,
42 crc64_msb_vpclmul_avx2);
43 static_call_update(crc64_lsb_pclmul,
44 crc64_lsb_vpclmul_avx2);
45 }
46 }
47 }
48 }
49