1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * x86-optimized CRC32 functions 4 * 5 * Copyright (C) 2008 Intel Corporation 6 * Copyright 2012 Xyratex Technology Limited 7 * Copyright 2024 Google LLC 8 */ 9 10 #include <asm/cpufeatures.h> 11 #include <asm/simd.h> 12 #include <crypto/internal/simd.h> 13 #include <linux/crc32.h> 14 #include <linux/linkage.h> 15 #include <linux/module.h> 16 17 /* minimum size of buffer for crc32_pclmul_le_16 */ 18 #define CRC32_PCLMUL_MIN_LEN 64 19 20 static DEFINE_STATIC_KEY_FALSE(have_crc32); 21 static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 22 23 u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len); 24 25 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 26 { 27 if (len >= CRC32_PCLMUL_MIN_LEN + 15 && 28 static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) { 29 size_t n = -(uintptr_t)p & 15; 30 31 /* align p to 16-byte boundary */ 32 if (n) { 33 crc = crc32_le_base(crc, p, n); 34 p += n; 35 len -= n; 36 } 37 n = round_down(len, 16); 38 kernel_fpu_begin(); 39 crc = crc32_pclmul_le_16(crc, p, n); 40 kernel_fpu_end(); 41 p += n; 42 len -= n; 43 } 44 if (len) 45 crc = crc32_le_base(crc, p, len); 46 return crc; 47 } 48 EXPORT_SYMBOL(crc32_le_arch); 49 50 #ifdef CONFIG_X86_64 51 #define CRC32_INST "crc32q %1, %q0" 52 #else 53 #define CRC32_INST "crc32l %1, %0" 54 #endif 55 56 /* 57 * Use carryless multiply version of crc32c when buffer size is >= 512 to 58 * account for FPU state save/restore overhead. 59 */ 60 #define CRC32C_PCLMUL_BREAKEVEN 512 61 62 asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len); 63 64 u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len) 65 { 66 size_t num_longs; 67 68 if (!static_branch_likely(&have_crc32)) 69 return crc32c_le_base(crc, p, len); 70 71 if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN && 72 static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) { 73 kernel_fpu_begin(); 74 crc = crc32c_x86_3way(crc, p, len); 75 kernel_fpu_end(); 76 return crc; 77 } 78 79 for (num_longs = len / sizeof(unsigned long); 80 num_longs != 0; num_longs--, p += sizeof(unsigned long)) 81 asm(CRC32_INST : "+r" (crc) : "rm" (*(unsigned long *)p)); 82 83 for (len %= sizeof(unsigned long); len; len--, p++) 84 asm("crc32b %1, %0" : "+r" (crc) : "rm" (*p)); 85 86 return crc; 87 } 88 EXPORT_SYMBOL(crc32c_le_arch); 89 90 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 91 { 92 return crc32_be_base(crc, p, len); 93 } 94 EXPORT_SYMBOL(crc32_be_arch); 95 96 static int __init crc32_x86_init(void) 97 { 98 if (boot_cpu_has(X86_FEATURE_XMM4_2)) 99 static_branch_enable(&have_crc32); 100 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) 101 static_branch_enable(&have_pclmulqdq); 102 return 0; 103 } 104 arch_initcall(crc32_x86_init); 105 106 static void __exit crc32_x86_exit(void) 107 { 108 } 109 module_exit(crc32_x86_exit); 110 111 u32 crc32_optimizations(void) 112 { 113 u32 optimizations = 0; 114 115 if (static_key_enabled(&have_crc32)) 116 optimizations |= CRC32C_OPTIMIZATION; 117 if (static_key_enabled(&have_pclmulqdq)) 118 optimizations |= CRC32_LE_OPTIMIZATION; 119 return optimizations; 120 } 121 EXPORT_SYMBOL(crc32_optimizations); 122 123 MODULE_DESCRIPTION("x86-optimized CRC32 functions"); 124 MODULE_LICENSE("GPL"); 125