1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * CRC32 and CRC32C using LoongArch crc* instructions
4 *
5 * Module based on mips/crypto/crc32-mips.c
6 *
7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8 * Copyright (C) 2018 MIPS Tech, LLC
9 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
10 */
11
12 #include <asm/cpu-features.h>
13 #include <linux/crc32.h>
14 #include <linux/module.h>
15 #include <linux/unaligned.h>
16
17 #define _CRC32(crc, value, size, type) \
18 do { \
19 __asm__ __volatile__( \
20 #type ".w." #size ".w" " %0, %1, %0\n\t"\
21 : "+r" (crc) \
22 : "r" (value) \
23 : "memory"); \
24 } while (0)
25
26 #define CRC32(crc, value, size) _CRC32(crc, value, size, crc)
27 #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc)
28
29 static DEFINE_STATIC_KEY_FALSE(have_crc32);
30
crc32_le_arch(u32 crc,const u8 * p,size_t len)31 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
32 {
33 if (!static_branch_likely(&have_crc32))
34 return crc32_le_base(crc, p, len);
35
36 while (len >= sizeof(u64)) {
37 u64 value = get_unaligned_le64(p);
38
39 CRC32(crc, value, d);
40 p += sizeof(u64);
41 len -= sizeof(u64);
42 }
43
44 if (len & sizeof(u32)) {
45 u32 value = get_unaligned_le32(p);
46
47 CRC32(crc, value, w);
48 p += sizeof(u32);
49 }
50
51 if (len & sizeof(u16)) {
52 u16 value = get_unaligned_le16(p);
53
54 CRC32(crc, value, h);
55 p += sizeof(u16);
56 }
57
58 if (len & sizeof(u8)) {
59 u8 value = *p++;
60
61 CRC32(crc, value, b);
62 }
63
64 return crc;
65 }
66 EXPORT_SYMBOL(crc32_le_arch);
67
crc32c_le_arch(u32 crc,const u8 * p,size_t len)68 u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
69 {
70 if (!static_branch_likely(&have_crc32))
71 return crc32c_le_base(crc, p, len);
72
73 while (len >= sizeof(u64)) {
74 u64 value = get_unaligned_le64(p);
75
76 CRC32C(crc, value, d);
77 p += sizeof(u64);
78 len -= sizeof(u64);
79 }
80
81 if (len & sizeof(u32)) {
82 u32 value = get_unaligned_le32(p);
83
84 CRC32C(crc, value, w);
85 p += sizeof(u32);
86 }
87
88 if (len & sizeof(u16)) {
89 u16 value = get_unaligned_le16(p);
90
91 CRC32C(crc, value, h);
92 p += sizeof(u16);
93 }
94
95 if (len & sizeof(u8)) {
96 u8 value = *p++;
97
98 CRC32C(crc, value, b);
99 }
100
101 return crc;
102 }
103 EXPORT_SYMBOL(crc32c_le_arch);
104
crc32_be_arch(u32 crc,const u8 * p,size_t len)105 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
106 {
107 return crc32_be_base(crc, p, len);
108 }
109 EXPORT_SYMBOL(crc32_be_arch);
110
crc32_loongarch_init(void)111 static int __init crc32_loongarch_init(void)
112 {
113 if (cpu_has_crc32)
114 static_branch_enable(&have_crc32);
115 return 0;
116 }
117 arch_initcall(crc32_loongarch_init);
118
crc32_loongarch_exit(void)119 static void __exit crc32_loongarch_exit(void)
120 {
121 }
122 module_exit(crc32_loongarch_exit);
123
crc32_optimizations(void)124 u32 crc32_optimizations(void)
125 {
126 if (static_key_enabled(&have_crc32))
127 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
128 return 0;
129 }
130 EXPORT_SYMBOL(crc32_optimizations);
131
132 MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
133 MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
134 MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
135 MODULE_LICENSE("GPL v2");
136