1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * CRC-32 implemented with the z/Architecture Vector Extension Facility.
4 *
5 * Copyright IBM Corp. 2015
6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7 */
8 #define KMSG_COMPONENT "crc32-vx"
9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
11 #include <linux/module.h>
12 #include <linux/cpufeature.h>
13 #include <linux/crc32.h>
14 #include <asm/fpu.h>
15 #include "crc32-vx.h"
16
17 #define VX_MIN_LEN 64
18 #define VX_ALIGNMENT 16L
19 #define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
20
21 static DEFINE_STATIC_KEY_FALSE(have_vxrs);
22
23 /*
24 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
25 *
26 * Creates a function to perform a particular CRC-32 computation. Depending
27 * on the message buffer, the hardware-accelerated or software implementation
28 * is used. Note that the message buffer is aligned to improve fetch
29 * operations of VECTOR LOAD MULTIPLE instructions.
30 */
31 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
32 u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
33 { \
34 unsigned long prealign, aligned, remaining; \
35 DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
36 \
37 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || \
38 !static_branch_likely(&have_vxrs)) \
39 return ___crc32_sw(crc, data, datalen); \
40 \
41 if ((unsigned long)data & VX_ALIGN_MASK) { \
42 prealign = VX_ALIGNMENT - \
43 ((unsigned long)data & VX_ALIGN_MASK); \
44 datalen -= prealign; \
45 crc = ___crc32_sw(crc, data, prealign); \
46 data = (void *)((unsigned long)data + prealign); \
47 } \
48 \
49 aligned = datalen & ~VX_ALIGN_MASK; \
50 remaining = datalen & VX_ALIGN_MASK; \
51 \
52 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
53 crc = ___crc32_vx(crc, data, aligned); \
54 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
55 \
56 if (remaining) \
57 crc = ___crc32_sw(crc, data + aligned, remaining); \
58 \
59 return crc; \
60 } \
61 EXPORT_SYMBOL(___fname);
62
DEFINE_CRC32_VX(crc32_le_arch,crc32_le_vgfm_16,crc32_le_base)63 DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
64 DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
65 DEFINE_CRC32_VX(crc32c_le_arch, crc32c_le_vgfm_16, crc32c_le_base)
66
67 static int __init crc32_s390_init(void)
68 {
69 if (cpu_have_feature(S390_CPU_FEATURE_VXRS))
70 static_branch_enable(&have_vxrs);
71 return 0;
72 }
73 arch_initcall(crc32_s390_init);
74
crc32_s390_exit(void)75 static void __exit crc32_s390_exit(void)
76 {
77 }
78 module_exit(crc32_s390_exit);
79
crc32_optimizations(void)80 u32 crc32_optimizations(void)
81 {
82 if (static_key_enabled(&have_vxrs))
83 return CRC32_LE_OPTIMIZATION |
84 CRC32_BE_OPTIMIZATION |
85 CRC32C_OPTIMIZATION;
86 return 0;
87 }
88 EXPORT_SYMBOL(crc32_optimizations);
89
90 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
91 MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
92 MODULE_LICENSE("GPL");
93