1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * x86-optimized CRC32 functions
4 *
5 * Copyright (C) 2008 Intel Corporation
6 * Copyright 2012 Xyratex Technology Limited
7 * Copyright 2024 Google LLC
8 */
9
10 #include <asm/cpufeatures.h>
11 #include <asm/simd.h>
12 #include <crypto/internal/simd.h>
13 #include <linux/crc32.h>
14 #include <linux/linkage.h>
15 #include <linux/module.h>
16
17 /* minimum size of buffer for crc32_pclmul_le_16 */
18 #define CRC32_PCLMUL_MIN_LEN 64
19
20 static DEFINE_STATIC_KEY_FALSE(have_crc32);
21 static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
22
23 u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
24
crc32_le_arch(u32 crc,const u8 * p,size_t len)25 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
26 {
27 if (len >= CRC32_PCLMUL_MIN_LEN + 15 &&
28 static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
29 size_t n = -(uintptr_t)p & 15;
30
31 /* align p to 16-byte boundary */
32 if (n) {
33 crc = crc32_le_base(crc, p, n);
34 p += n;
35 len -= n;
36 }
37 n = round_down(len, 16);
38 kernel_fpu_begin();
39 crc = crc32_pclmul_le_16(crc, p, n);
40 kernel_fpu_end();
41 p += n;
42 len -= n;
43 }
44 if (len)
45 crc = crc32_le_base(crc, p, len);
46 return crc;
47 }
48 EXPORT_SYMBOL(crc32_le_arch);
49
50 #ifdef CONFIG_X86_64
51 #define CRC32_INST "crc32q %1, %q0"
52 #else
53 #define CRC32_INST "crc32l %1, %0"
54 #endif
55
56 /*
57 * Use carryless multiply version of crc32c when buffer size is >= 512 to
58 * account for FPU state save/restore overhead.
59 */
60 #define CRC32C_PCLMUL_BREAKEVEN 512
61
62 asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
63
crc32c_le_arch(u32 crc,const u8 * p,size_t len)64 u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
65 {
66 size_t num_longs;
67
68 if (!static_branch_likely(&have_crc32))
69 return crc32c_le_base(crc, p, len);
70
71 if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
72 static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
73 kernel_fpu_begin();
74 crc = crc32c_x86_3way(crc, p, len);
75 kernel_fpu_end();
76 return crc;
77 }
78
79 for (num_longs = len / sizeof(unsigned long);
80 num_longs != 0; num_longs--, p += sizeof(unsigned long))
81 asm(CRC32_INST : "+r" (crc) : "rm" (*(unsigned long *)p));
82
83 for (len %= sizeof(unsigned long); len; len--, p++)
84 asm("crc32b %1, %0" : "+r" (crc) : "rm" (*p));
85
86 return crc;
87 }
88 EXPORT_SYMBOL(crc32c_le_arch);
89
crc32_be_arch(u32 crc,const u8 * p,size_t len)90 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
91 {
92 return crc32_be_base(crc, p, len);
93 }
94 EXPORT_SYMBOL(crc32_be_arch);
95
crc32_x86_init(void)96 static int __init crc32_x86_init(void)
97 {
98 if (boot_cpu_has(X86_FEATURE_XMM4_2))
99 static_branch_enable(&have_crc32);
100 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ))
101 static_branch_enable(&have_pclmulqdq);
102 return 0;
103 }
104 arch_initcall(crc32_x86_init);
105
crc32_x86_exit(void)106 static void __exit crc32_x86_exit(void)
107 {
108 }
109 module_exit(crc32_x86_exit);
110
crc32_optimizations(void)111 u32 crc32_optimizations(void)
112 {
113 u32 optimizations = 0;
114
115 if (static_key_enabled(&have_crc32))
116 optimizations |= CRC32C_OPTIMIZATION;
117 if (static_key_enabled(&have_pclmulqdq))
118 optimizations |= CRC32_LE_OPTIMIZATION;
119 return optimizations;
120 }
121 EXPORT_SYMBOL(crc32_optimizations);
122
123 MODULE_DESCRIPTION("x86-optimized CRC32 functions");
124 MODULE_LICENSE("GPL");
125