1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2024 Institute of Software, CAS.
4 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
5 */
6
7 #include <asm/vector.h>
8 #include <linux/raid/pq.h>
9
rvv_has_vector(void)10 static int rvv_has_vector(void)
11 {
12 return has_vector();
13 }
14
__raid6_2data_recov_rvv(int bytes,u8 * p,u8 * q,u8 * dp,u8 * dq,const u8 * pbmul,const u8 * qmul)15 static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
16 u8 *dq, const u8 *pbmul,
17 const u8 *qmul)
18 {
19 asm volatile (".option push\n"
20 ".option arch,+v\n"
21 "vsetvli x0, %[avl], e8, m1, ta, ma\n"
22 ".option pop\n"
23 : :
24 [avl]"r"(16)
25 );
26
27 /*
28 * while ( bytes-- ) {
29 * uint8_t px, qx, db;
30 *
31 * px = *p ^ *dp;
32 * qx = qmul[*q ^ *dq];
33 * *dq++ = db = pbmul[px] ^ qx;
34 * *dp++ = db ^ px;
35 * p++; q++;
36 * }
37 */
38 while (bytes) {
39 /*
40 * v0:px, v1:dp,
41 * v2:qx, v3:dq,
42 * v4:vx, v5:vy,
43 * v6:qm0, v7:qm1,
44 * v8:pm0, v9:pm1,
45 * v14:p/qm[vx], v15:p/qm[vy]
46 */
47 asm volatile (".option push\n"
48 ".option arch,+v\n"
49 "vle8.v v0, (%[px])\n"
50 "vle8.v v1, (%[dp])\n"
51 "vxor.vv v0, v0, v1\n"
52 "vle8.v v2, (%[qx])\n"
53 "vle8.v v3, (%[dq])\n"
54 "vxor.vv v4, v2, v3\n"
55 "vsrl.vi v5, v4, 4\n"
56 "vand.vi v4, v4, 0xf\n"
57 "vle8.v v6, (%[qm0])\n"
58 "vle8.v v7, (%[qm1])\n"
59 "vrgather.vv v14, v6, v4\n" /* v14 = qm[vx] */
60 "vrgather.vv v15, v7, v5\n" /* v15 = qm[vy] */
61 "vxor.vv v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */
62
63 "vsrl.vi v5, v0, 4\n"
64 "vand.vi v4, v0, 0xf\n"
65 "vle8.v v8, (%[pm0])\n"
66 "vle8.v v9, (%[pm1])\n"
67 "vrgather.vv v14, v8, v4\n" /* v14 = pm[vx] */
68 "vrgather.vv v15, v9, v5\n" /* v15 = pm[vy] */
69 "vxor.vv v4, v14, v15\n" /* v4 = pbmul[px] */
70 "vxor.vv v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */
71 "vxor.vv v1, v3, v0\n" /* v1 = db ^ px; */
72 "vse8.v v3, (%[dq])\n"
73 "vse8.v v1, (%[dp])\n"
74 ".option pop\n"
75 : :
76 [px]"r"(p),
77 [dp]"r"(dp),
78 [qx]"r"(q),
79 [dq]"r"(dq),
80 [qm0]"r"(qmul),
81 [qm1]"r"(qmul + 16),
82 [pm0]"r"(pbmul),
83 [pm1]"r"(pbmul + 16)
84 :);
85
86 bytes -= 16;
87 p += 16;
88 q += 16;
89 dp += 16;
90 dq += 16;
91 }
92 }
93
__raid6_datap_recov_rvv(int bytes,u8 * p,u8 * q,u8 * dq,const u8 * qmul)94 static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q,
95 u8 *dq, const u8 *qmul)
96 {
97 asm volatile (".option push\n"
98 ".option arch,+v\n"
99 "vsetvli x0, %[avl], e8, m1, ta, ma\n"
100 ".option pop\n"
101 : :
102 [avl]"r"(16)
103 );
104
105 /*
106 * while (bytes--) {
107 * *p++ ^= *dq = qmul[*q ^ *dq];
108 * q++; dq++;
109 * }
110 */
111 while (bytes) {
112 /*
113 * v0:vx, v1:vy,
114 * v2:dq, v3:p,
115 * v4:qm0, v5:qm1,
116 * v10:m[vx], v11:m[vy]
117 */
118 asm volatile (".option push\n"
119 ".option arch,+v\n"
120 "vle8.v v0, (%[vx])\n"
121 "vle8.v v2, (%[dq])\n"
122 "vxor.vv v0, v0, v2\n"
123 "vsrl.vi v1, v0, 4\n"
124 "vand.vi v0, v0, 0xf\n"
125 "vle8.v v4, (%[qm0])\n"
126 "vle8.v v5, (%[qm1])\n"
127 "vrgather.vv v10, v4, v0\n"
128 "vrgather.vv v11, v5, v1\n"
129 "vxor.vv v0, v10, v11\n"
130 "vle8.v v1, (%[vy])\n"
131 "vxor.vv v1, v0, v1\n"
132 "vse8.v v0, (%[dq])\n"
133 "vse8.v v1, (%[vy])\n"
134 ".option pop\n"
135 : :
136 [vx]"r"(q),
137 [vy]"r"(p),
138 [dq]"r"(dq),
139 [qm0]"r"(qmul),
140 [qm1]"r"(qmul + 16)
141 :);
142
143 bytes -= 16;
144 p += 16;
145 q += 16;
146 dq += 16;
147 }
148 }
149
raid6_2data_recov_rvv(int disks,size_t bytes,int faila,int failb,void ** ptrs)150 static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
151 int failb, void **ptrs)
152 {
153 u8 *p, *q, *dp, *dq;
154 const u8 *pbmul; /* P multiplier table for B data */
155 const u8 *qmul; /* Q multiplier table (for both) */
156
157 p = (u8 *)ptrs[disks - 2];
158 q = (u8 *)ptrs[disks - 1];
159
160 /*
161 * Compute syndrome with zero for the missing data pages
162 * Use the dead data pages as temporary storage for
163 * delta p and delta q
164 */
165 dp = (u8 *)ptrs[faila];
166 ptrs[faila] = raid6_get_zero_page();
167 ptrs[disks - 2] = dp;
168 dq = (u8 *)ptrs[failb];
169 ptrs[failb] = raid6_get_zero_page();
170 ptrs[disks - 1] = dq;
171
172 raid6_call.gen_syndrome(disks, bytes, ptrs);
173
174 /* Restore pointer table */
175 ptrs[faila] = dp;
176 ptrs[failb] = dq;
177 ptrs[disks - 2] = p;
178 ptrs[disks - 1] = q;
179
180 /* Now, pick the proper data tables */
181 pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
182 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
183 raid6_gfexp[failb]]];
184
185 kernel_vector_begin();
186 __raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul);
187 kernel_vector_end();
188 }
189
raid6_datap_recov_rvv(int disks,size_t bytes,int faila,void ** ptrs)190 static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
191 void **ptrs)
192 {
193 u8 *p, *q, *dq;
194 const u8 *qmul; /* Q multiplier table */
195
196 p = (u8 *)ptrs[disks - 2];
197 q = (u8 *)ptrs[disks - 1];
198
199 /*
200 * Compute syndrome with zero for the missing data page
201 * Use the dead data page as temporary storage for delta q
202 */
203 dq = (u8 *)ptrs[faila];
204 ptrs[faila] = raid6_get_zero_page();
205 ptrs[disks - 1] = dq;
206
207 raid6_call.gen_syndrome(disks, bytes, ptrs);
208
209 /* Restore pointer table */
210 ptrs[faila] = dq;
211 ptrs[disks - 1] = q;
212
213 /* Now, pick the proper data tables */
214 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
215
216 kernel_vector_begin();
217 __raid6_datap_recov_rvv(bytes, p, q, dq, qmul);
218 kernel_vector_end();
219 }
220
221 const struct raid6_recov_calls raid6_recov_rvv = {
222 .data2 = raid6_2data_recov_rvv,
223 .datap = raid6_datap_recov_rvv,
224 .valid = rvv_has_vector,
225 .name = "rvv",
226 .priority = 1,
227 };
228