1*ed4bc981SEric Biggers // SPDX-License-Identifier: GPL-2.0-or-later 2*ed4bc981SEric Biggers /* 3*ed4bc981SEric Biggers * CRC-T10DIF using PCLMULQDQ instructions 4*ed4bc981SEric Biggers * 5*ed4bc981SEric Biggers * Copyright 2024 Google LLC 6*ed4bc981SEric Biggers */ 7*ed4bc981SEric Biggers 8*ed4bc981SEric Biggers #include <asm/cpufeatures.h> 9*ed4bc981SEric Biggers #include <asm/simd.h> 10*ed4bc981SEric Biggers #include <crypto/internal/simd.h> 11*ed4bc981SEric Biggers #include <linux/crc-t10dif.h> 12*ed4bc981SEric Biggers #include <linux/module.h> 13*ed4bc981SEric Biggers 14*ed4bc981SEric Biggers static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 15*ed4bc981SEric Biggers 16*ed4bc981SEric Biggers asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len); 17*ed4bc981SEric Biggers 18*ed4bc981SEric Biggers u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len) 19*ed4bc981SEric Biggers { 20*ed4bc981SEric Biggers if (len >= 16 && 21*ed4bc981SEric Biggers static_key_enabled(&have_pclmulqdq) && crypto_simd_usable()) { 22*ed4bc981SEric Biggers kernel_fpu_begin(); 23*ed4bc981SEric Biggers crc = crc_t10dif_pcl(crc, p, len); 24*ed4bc981SEric Biggers kernel_fpu_end(); 25*ed4bc981SEric Biggers return crc; 26*ed4bc981SEric Biggers } 27*ed4bc981SEric Biggers return crc_t10dif_generic(crc, p, len); 28*ed4bc981SEric Biggers } 29*ed4bc981SEric Biggers EXPORT_SYMBOL(crc_t10dif_arch); 30*ed4bc981SEric Biggers 31*ed4bc981SEric Biggers static int __init crc_t10dif_x86_init(void) 32*ed4bc981SEric Biggers { 33*ed4bc981SEric Biggers if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) 34*ed4bc981SEric Biggers static_branch_enable(&have_pclmulqdq); 35*ed4bc981SEric Biggers return 0; 36*ed4bc981SEric Biggers } 37*ed4bc981SEric Biggers arch_initcall(crc_t10dif_x86_init); 38*ed4bc981SEric Biggers 39*ed4bc981SEric Biggers static void __exit crc_t10dif_x86_exit(void) 40*ed4bc981SEric Biggers { 41*ed4bc981SEric Biggers } 42*ed4bc981SEric Biggers module_exit(crc_t10dif_x86_exit); 43*ed4bc981SEric Biggers 44*ed4bc981SEric Biggers bool crc_t10dif_is_optimized(void) 45*ed4bc981SEric Biggers { 46*ed4bc981SEric Biggers return static_key_enabled(&have_pclmulqdq); 47*ed4bc981SEric Biggers } 48*ed4bc981SEric Biggers EXPORT_SYMBOL(crc_t10dif_is_optimized); 49*ed4bc981SEric Biggers 50*ed4bc981SEric Biggers MODULE_DESCRIPTION("CRC-T10DIF using PCLMULQDQ instructions"); 51*ed4bc981SEric Biggers MODULE_LICENSE("GPL"); 52