1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * x86-optimized CRC32 functions 4 * 5 * Copyright (C) 2008 Intel Corporation 6 * Copyright 2012 Xyratex Technology Limited 7 * Copyright 2024 Google LLC 8 */ 9 10 #include <linux/crc32.h> 11 #include <linux/module.h> 12 #include "crc-pclmul-template.h" 13 14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 16 17 DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32); 18 19 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 20 { 21 CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts, 22 have_pclmulqdq); 23 return crc32_le_base(crc, p, len); 24 } 25 EXPORT_SYMBOL(crc32_le_arch); 26 27 #ifdef CONFIG_X86_64 28 #define CRC32_INST "crc32q %1, %q0" 29 #else 30 #define CRC32_INST "crc32l %1, %0" 31 #endif 32 33 /* 34 * Use carryless multiply version of crc32c when buffer size is >= 512 to 35 * account for FPU state save/restore overhead. 36 */ 37 #define CRC32C_PCLMUL_BREAKEVEN 512 38 39 asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len); 40 41 u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 42 { 43 size_t num_longs; 44 45 if (!static_branch_likely(&have_crc32)) 46 return crc32c_base(crc, p, len); 47 48 if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN && 49 static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) { 50 kernel_fpu_begin(); 51 crc = crc32c_x86_3way(crc, p, len); 52 kernel_fpu_end(); 53 return crc; 54 } 55 56 for (num_longs = len / sizeof(unsigned long); 57 num_longs != 0; num_longs--, p += sizeof(unsigned long)) 58 asm(CRC32_INST : "+r" (crc) : ASM_INPUT_RM (*(unsigned long *)p)); 59 60 if (sizeof(unsigned long) > 4 && (len & 4)) { 61 asm("crc32l %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u32 *)p)); 62 p += 4; 63 } 64 if (len & 2) { 65 asm("crc32w %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u16 *)p)); 66 p += 2; 67 } 68 if (len & 1) 69 asm("crc32b %1, %0" : "+r" (crc) : ASM_INPUT_RM (*p)); 70 71 return crc; 72 } 73 EXPORT_SYMBOL(crc32c_arch); 74 75 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 76 { 77 return crc32_be_base(crc, p, len); 78 } 79 EXPORT_SYMBOL(crc32_be_arch); 80 81 static int __init crc32_x86_init(void) 82 { 83 if (boot_cpu_has(X86_FEATURE_XMM4_2)) 84 static_branch_enable(&have_crc32); 85 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 86 static_branch_enable(&have_pclmulqdq); 87 INIT_CRC_PCLMUL(crc32_lsb); 88 } 89 return 0; 90 } 91 subsys_initcall(crc32_x86_init); 92 93 static void __exit crc32_x86_exit(void) 94 { 95 } 96 module_exit(crc32_x86_exit); 97 98 u32 crc32_optimizations(void) 99 { 100 u32 optimizations = 0; 101 102 if (static_key_enabled(&have_crc32)) 103 optimizations |= CRC32C_OPTIMIZATION; 104 if (static_key_enabled(&have_pclmulqdq)) 105 optimizations |= CRC32_LE_OPTIMIZATION; 106 return optimizations; 107 } 108 EXPORT_SYMBOL(crc32_optimizations); 109 110 MODULE_DESCRIPTION("x86-optimized CRC32 functions"); 111 MODULE_LICENSE("GPL"); 112