xref: /linux/lib/raid/raid6/riscv/rvv.c (revision 26ba30221c03364d6ed9910be8da4c1fd871b07b) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * RAID-6 syndrome calculation using RISC-V vector instructions
4  *
5  * Copyright 2024 Institute of Software, CAS.
6  * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
7  *
8  * Based on neon.uc:
9  *	Copyright 2002-2004 H. Peter Anvin
10  */
11 
12 #include "rvv.h"
13 #include "pq_arch.h"
14 
15 #ifdef __riscv_vector
16 #error "This code must be built without compiler support for vector"
17 #endif
18 
19 static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
20 {
21 	u8 **dptr = (u8 **)ptrs;
22 	u8 *p, *q;
23 	unsigned long vl, d, nsize;
24 	int z, z0;
25 
26 	z0 = disks - 3;		/* Highest data disk */
27 	p = dptr[z0 + 1];		/* XOR parity */
28 	q = dptr[z0 + 2];		/* RS syndrome */
29 
30 	asm volatile (".option	push\n"
31 		      ".option	arch,+v\n"
32 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
33 		      ".option	pop\n"
34 		      : "=&r" (vl)
35 	);
36 
37 	nsize = vl;
38 
39 	 /* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
40 	for (d = 0; d < bytes; d += nsize * 1) {
41 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
42 		asm volatile (".option	push\n"
43 			      ".option	arch,+v\n"
44 			      "vle8.v	v0, (%[wp0])\n"
45 			      "vmv.v.v	v1, v0\n"
46 			      ".option	pop\n"
47 			      : :
48 			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
49 		);
50 
51 		for (z = z0 - 1 ; z >= 0 ; z--) {
52 			/*
53 			 * w2$$ = MASK(wq$$);
54 			 * w1$$ = SHLBYTE(wq$$);
55 			 * w2$$ &= NBYTES(0x1d);
56 			 * w1$$ ^= w2$$;
57 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
58 			 * wq$$ = w1$$ ^ wd$$;
59 			 * wp$$ ^= wd$$;
60 			 */
61 			asm volatile (".option	push\n"
62 				      ".option	arch,+v\n"
63 				      "vsra.vi	v2, v1, 7\n"
64 				      "vsll.vi	v3, v1, 1\n"
65 				      "vand.vx	v2, v2, %[x1d]\n"
66 				      "vxor.vv	v3, v3, v2\n"
67 				      "vle8.v	v2, (%[wd0])\n"
68 				      "vxor.vv	v1, v3, v2\n"
69 				      "vxor.vv	v0, v0, v2\n"
70 				      ".option	pop\n"
71 				      : :
72 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
73 				      [x1d]"r"(0x1d)
74 			);
75 		}
76 
77 		/*
78 		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
79 		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
80 		 */
81 		asm volatile (".option	push\n"
82 			      ".option	arch,+v\n"
83 			      "vse8.v	v0, (%[wp0])\n"
84 			      "vse8.v	v1, (%[wq0])\n"
85 			      ".option	pop\n"
86 			      : :
87 			      [wp0]"r"(&p[d + nsize * 0]),
88 			      [wq0]"r"(&q[d + nsize * 0])
89 		);
90 	}
91 }
92 
93 static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
94 					 unsigned long bytes, void **ptrs)
95 {
96 	u8 **dptr = (u8 **)ptrs;
97 	u8 *p, *q;
98 	unsigned long vl, d, nsize;
99 	int z, z0;
100 
101 	z0 = stop;		/* P/Q right side optimization */
102 	p = dptr[disks - 2];	/* XOR parity */
103 	q = dptr[disks - 1];	/* RS syndrome */
104 
105 	asm volatile (".option	push\n"
106 		      ".option	arch,+v\n"
107 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
108 		      ".option	pop\n"
109 		      : "=&r" (vl)
110 	);
111 
112 	nsize = vl;
113 
114 	/* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
115 	for (d = 0 ; d < bytes ; d += nsize * 1) {
116 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
117 		asm volatile (".option	push\n"
118 			      ".option	arch,+v\n"
119 			      "vle8.v	v0, (%[wp0])\n"
120 			      "vmv.v.v	v1, v0\n"
121 			      ".option	pop\n"
122 			      : :
123 			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
124 		);
125 
126 		/* P/Q data pages */
127 		for (z = z0 - 1; z >= start; z--) {
128 			/*
129 			 * w2$$ = MASK(wq$$);
130 			 * w1$$ = SHLBYTE(wq$$);
131 			 * w2$$ &= NBYTES(0x1d);
132 			 * w1$$ ^= w2$$;
133 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
134 			 * wq$$ = w1$$ ^ wd$$;
135 			 * wp$$ ^= wd$$;
136 			 */
137 			asm volatile (".option	push\n"
138 				      ".option	arch,+v\n"
139 				      "vsra.vi	v2, v1, 7\n"
140 				      "vsll.vi	v3, v1, 1\n"
141 				      "vand.vx	v2, v2, %[x1d]\n"
142 				      "vxor.vv	v3, v3, v2\n"
143 				      "vle8.v	v2, (%[wd0])\n"
144 				      "vxor.vv	v1, v3, v2\n"
145 				      "vxor.vv	v0, v0, v2\n"
146 				      ".option	pop\n"
147 				      : :
148 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
149 				      [x1d]"r"(0x1d)
150 			);
151 		}
152 
153 		/* P/Q left side optimization */
154 		for (z = start - 1; z >= 0; z--) {
155 			/*
156 			 * w2$$ = MASK(wq$$);
157 			 * w1$$ = SHLBYTE(wq$$);
158 			 * w2$$ &= NBYTES(0x1d);
159 			 * wq$$ = w1$$ ^ w2$$;
160 			 */
161 			asm volatile (".option	push\n"
162 				      ".option	arch,+v\n"
163 				      "vsra.vi	v2, v1, 7\n"
164 				      "vsll.vi	v3, v1, 1\n"
165 				      "vand.vx	v2, v2, %[x1d]\n"
166 				      "vxor.vv	v1, v3, v2\n"
167 				      ".option	pop\n"
168 				      : :
169 				      [x1d]"r"(0x1d)
170 			);
171 		}
172 
173 		/*
174 		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
175 		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
176 		 * v0:wp0, v1:wq0, v2:p0, v3:q0
177 		 */
178 		asm volatile (".option	push\n"
179 			      ".option	arch,+v\n"
180 			      "vle8.v	v2, (%[wp0])\n"
181 			      "vle8.v	v3, (%[wq0])\n"
182 			      "vxor.vv	v2, v2, v0\n"
183 			      "vxor.vv	v3, v3, v1\n"
184 			      "vse8.v	v2, (%[wp0])\n"
185 			      "vse8.v	v3, (%[wq0])\n"
186 			      ".option	pop\n"
187 			      : :
188 			      [wp0]"r"(&p[d + nsize * 0]),
189 			      [wq0]"r"(&q[d + nsize * 0])
190 		);
191 	}
192 }
193 
194 static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
195 {
196 	u8 **dptr = (u8 **)ptrs;
197 	u8 *p, *q;
198 	unsigned long vl, d, nsize;
199 	int z, z0;
200 
201 	z0 = disks - 3;		/* Highest data disk */
202 	p = dptr[z0 + 1];		/* XOR parity */
203 	q = dptr[z0 + 2];		/* RS syndrome */
204 
205 	asm volatile (".option	push\n"
206 		      ".option	arch,+v\n"
207 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
208 		      ".option	pop\n"
209 		      : "=&r" (vl)
210 	);
211 
212 	nsize = vl;
213 
214 	/*
215 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
216 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
217 	 */
218 	for (d = 0; d < bytes; d += nsize * 2) {
219 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
220 		asm volatile (".option	push\n"
221 			      ".option	arch,+v\n"
222 			      "vle8.v	v0, (%[wp0])\n"
223 			      "vmv.v.v	v1, v0\n"
224 			      "vle8.v	v4, (%[wp1])\n"
225 			      "vmv.v.v	v5, v4\n"
226 			      ".option	pop\n"
227 			      : :
228 			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
229 			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
230 		);
231 
232 		for (z = z0 - 1; z >= 0; z--) {
233 			/*
234 			 * w2$$ = MASK(wq$$);
235 			 * w1$$ = SHLBYTE(wq$$);
236 			 * w2$$ &= NBYTES(0x1d);
237 			 * w1$$ ^= w2$$;
238 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
239 			 * wq$$ = w1$$ ^ wd$$;
240 			 * wp$$ ^= wd$$;
241 			 */
242 			asm volatile (".option	push\n"
243 				      ".option	arch,+v\n"
244 				      "vsra.vi	v2, v1, 7\n"
245 				      "vsll.vi	v3, v1, 1\n"
246 				      "vand.vx	v2, v2, %[x1d]\n"
247 				      "vxor.vv	v3, v3, v2\n"
248 				      "vle8.v	v2, (%[wd0])\n"
249 				      "vxor.vv	v1, v3, v2\n"
250 				      "vxor.vv	v0, v0, v2\n"
251 
252 				      "vsra.vi	v6, v5, 7\n"
253 				      "vsll.vi	v7, v5, 1\n"
254 				      "vand.vx	v6, v6, %[x1d]\n"
255 				      "vxor.vv	v7, v7, v6\n"
256 				      "vle8.v	v6, (%[wd1])\n"
257 				      "vxor.vv	v5, v7, v6\n"
258 				      "vxor.vv	v4, v4, v6\n"
259 				      ".option	pop\n"
260 				      : :
261 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
262 				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
263 				      [x1d]"r"(0x1d)
264 			);
265 		}
266 
267 		/*
268 		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
269 		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
270 		 */
271 		asm volatile (".option	push\n"
272 			      ".option	arch,+v\n"
273 			      "vse8.v	v0, (%[wp0])\n"
274 			      "vse8.v	v1, (%[wq0])\n"
275 			      "vse8.v	v4, (%[wp1])\n"
276 			      "vse8.v	v5, (%[wq1])\n"
277 			      ".option	pop\n"
278 			      : :
279 			      [wp0]"r"(&p[d + nsize * 0]),
280 			      [wq0]"r"(&q[d + nsize * 0]),
281 			      [wp1]"r"(&p[d + nsize * 1]),
282 			      [wq1]"r"(&q[d + nsize * 1])
283 		);
284 	}
285 }
286 
287 static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
288 					 unsigned long bytes, void **ptrs)
289 {
290 	u8 **dptr = (u8 **)ptrs;
291 	u8 *p, *q;
292 	unsigned long vl, d, nsize;
293 	int z, z0;
294 
295 	z0 = stop;		/* P/Q right side optimization */
296 	p = dptr[disks - 2];	/* XOR parity */
297 	q = dptr[disks - 1];	/* RS syndrome */
298 
299 	asm volatile (".option	push\n"
300 		      ".option	arch,+v\n"
301 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
302 		      ".option	pop\n"
303 		      : "=&r" (vl)
304 	);
305 
306 	nsize = vl;
307 
308 	/*
309 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
310 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
311 	 */
312 	for (d = 0; d < bytes; d += nsize * 2) {
313 		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
314 		asm volatile (".option	push\n"
315 			      ".option	arch,+v\n"
316 			      "vle8.v	v0, (%[wp0])\n"
317 			      "vmv.v.v	v1, v0\n"
318 			      "vle8.v	v4, (%[wp1])\n"
319 			      "vmv.v.v	v5, v4\n"
320 			      ".option	pop\n"
321 			      : :
322 			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
323 			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
324 		);
325 
326 		/* P/Q data pages */
327 		for (z = z0 - 1; z >= start; z--) {
328 			/*
329 			 * w2$$ = MASK(wq$$);
330 			 * w1$$ = SHLBYTE(wq$$);
331 			 * w2$$ &= NBYTES(0x1d);
332 			 * w1$$ ^= w2$$;
333 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
334 			 * wq$$ = w1$$ ^ wd$$;
335 			 * wp$$ ^= wd$$;
336 			 */
337 			asm volatile (".option	push\n"
338 				      ".option	arch,+v\n"
339 				      "vsra.vi	v2, v1, 7\n"
340 				      "vsll.vi	v3, v1, 1\n"
341 				      "vand.vx	v2, v2, %[x1d]\n"
342 				      "vxor.vv	v3, v3, v2\n"
343 				      "vle8.v	v2, (%[wd0])\n"
344 				      "vxor.vv	v1, v3, v2\n"
345 				      "vxor.vv	v0, v0, v2\n"
346 
347 				      "vsra.vi	v6, v5, 7\n"
348 				      "vsll.vi	v7, v5, 1\n"
349 				      "vand.vx	v6, v6, %[x1d]\n"
350 				      "vxor.vv	v7, v7, v6\n"
351 				      "vle8.v	v6, (%[wd1])\n"
352 				      "vxor.vv	v5, v7, v6\n"
353 				      "vxor.vv	v4, v4, v6\n"
354 				      ".option	pop\n"
355 				      : :
356 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
357 				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
358 				      [x1d]"r"(0x1d)
359 			);
360 		}
361 
362 		/* P/Q left side optimization */
363 		for (z = start - 1; z >= 0; z--) {
364 			/*
365 			 * w2$$ = MASK(wq$$);
366 			 * w1$$ = SHLBYTE(wq$$);
367 			 * w2$$ &= NBYTES(0x1d);
368 			 * wq$$ = w1$$ ^ w2$$;
369 			 */
370 			asm volatile (".option	push\n"
371 				      ".option	arch,+v\n"
372 				      "vsra.vi	v2, v1, 7\n"
373 				      "vsll.vi	v3, v1, 1\n"
374 				      "vand.vx	v2, v2, %[x1d]\n"
375 				      "vxor.vv	v1, v3, v2\n"
376 
377 				      "vsra.vi	v6, v5, 7\n"
378 				      "vsll.vi	v7, v5, 1\n"
379 				      "vand.vx	v6, v6, %[x1d]\n"
380 				      "vxor.vv	v5, v7, v6\n"
381 				      ".option	pop\n"
382 				      : :
383 				      [x1d]"r"(0x1d)
384 			);
385 		}
386 
387 		/*
388 		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
389 		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
390 		 * v0:wp0, v1:wq0, v2:p0, v3:q0
391 		 * v4:wp1, v5:wq1, v6:p1, v7:q1
392 		 */
393 		asm volatile (".option	push\n"
394 			      ".option	arch,+v\n"
395 			      "vle8.v	v2, (%[wp0])\n"
396 			      "vle8.v	v3, (%[wq0])\n"
397 			      "vxor.vv	v2, v2, v0\n"
398 			      "vxor.vv	v3, v3, v1\n"
399 			      "vse8.v	v2, (%[wp0])\n"
400 			      "vse8.v	v3, (%[wq0])\n"
401 
402 			      "vle8.v	v6, (%[wp1])\n"
403 			      "vle8.v	v7, (%[wq1])\n"
404 			      "vxor.vv	v6, v6, v4\n"
405 			      "vxor.vv	v7, v7, v5\n"
406 			      "vse8.v	v6, (%[wp1])\n"
407 			      "vse8.v	v7, (%[wq1])\n"
408 			      ".option	pop\n"
409 			      : :
410 			      [wp0]"r"(&p[d + nsize * 0]),
411 			      [wq0]"r"(&q[d + nsize * 0]),
412 			      [wp1]"r"(&p[d + nsize * 1]),
413 			      [wq1]"r"(&q[d + nsize * 1])
414 		);
415 	}
416 }
417 
418 static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
419 {
420 	u8 **dptr = (u8 **)ptrs;
421 	u8 *p, *q;
422 	unsigned long vl, d, nsize;
423 	int z, z0;
424 
425 	z0 = disks - 3;	/* Highest data disk */
426 	p = dptr[z0 + 1];	/* XOR parity */
427 	q = dptr[z0 + 2];	/* RS syndrome */
428 
429 	asm volatile (".option	push\n"
430 		      ".option	arch,+v\n"
431 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
432 		      ".option	pop\n"
433 		      : "=&r" (vl)
434 	);
435 
436 	nsize = vl;
437 
438 	/*
439 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
440 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
441 	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
442 	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
443 	 */
444 	for (d = 0; d < bytes; d += nsize * 4) {
445 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
446 		asm volatile (".option	push\n"
447 			      ".option	arch,+v\n"
448 			      "vle8.v	v0, (%[wp0])\n"
449 			      "vmv.v.v	v1, v0\n"
450 			      "vle8.v	v4, (%[wp1])\n"
451 			      "vmv.v.v	v5, v4\n"
452 			      "vle8.v	v8, (%[wp2])\n"
453 			      "vmv.v.v	v9, v8\n"
454 			      "vle8.v	v12, (%[wp3])\n"
455 			      "vmv.v.v	v13, v12\n"
456 			      ".option	pop\n"
457 			      : :
458 			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
459 			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
460 			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
461 			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
462 		);
463 
464 		for (z = z0 - 1; z >= 0; z--) {
465 			/*
466 			 * w2$$ = MASK(wq$$);
467 			 * w1$$ = SHLBYTE(wq$$);
468 			 * w2$$ &= NBYTES(0x1d);
469 			 * w1$$ ^= w2$$;
470 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
471 			 * wq$$ = w1$$ ^ wd$$;
472 			 * wp$$ ^= wd$$;
473 			 */
474 			asm volatile (".option	push\n"
475 				      ".option	arch,+v\n"
476 				      "vsra.vi	v2, v1, 7\n"
477 				      "vsll.vi	v3, v1, 1\n"
478 				      "vand.vx	v2, v2, %[x1d]\n"
479 				      "vxor.vv	v3, v3, v2\n"
480 				      "vle8.v	v2, (%[wd0])\n"
481 				      "vxor.vv	v1, v3, v2\n"
482 				      "vxor.vv	v0, v0, v2\n"
483 
484 				      "vsra.vi	v6, v5, 7\n"
485 				      "vsll.vi	v7, v5, 1\n"
486 				      "vand.vx	v6, v6, %[x1d]\n"
487 				      "vxor.vv	v7, v7, v6\n"
488 				      "vle8.v	v6, (%[wd1])\n"
489 				      "vxor.vv	v5, v7, v6\n"
490 				      "vxor.vv	v4, v4, v6\n"
491 
492 				      "vsra.vi	v10, v9, 7\n"
493 				      "vsll.vi	v11, v9, 1\n"
494 				      "vand.vx	v10, v10, %[x1d]\n"
495 				      "vxor.vv	v11, v11, v10\n"
496 				      "vle8.v	v10, (%[wd2])\n"
497 				      "vxor.vv	v9, v11, v10\n"
498 				      "vxor.vv	v8, v8, v10\n"
499 
500 				      "vsra.vi	v14, v13, 7\n"
501 				      "vsll.vi	v15, v13, 1\n"
502 				      "vand.vx	v14, v14, %[x1d]\n"
503 				      "vxor.vv	v15, v15, v14\n"
504 				      "vle8.v	v14, (%[wd3])\n"
505 				      "vxor.vv	v13, v15, v14\n"
506 				      "vxor.vv	v12, v12, v14\n"
507 				      ".option	pop\n"
508 				      : :
509 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
510 				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
511 				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
512 				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
513 				      [x1d]"r"(0x1d)
514 			);
515 		}
516 
517 		/*
518 		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
519 		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
520 		 */
521 		asm volatile (".option	push\n"
522 			      ".option	arch,+v\n"
523 			      "vse8.v	v0, (%[wp0])\n"
524 			      "vse8.v	v1, (%[wq0])\n"
525 			      "vse8.v	v4, (%[wp1])\n"
526 			      "vse8.v	v5, (%[wq1])\n"
527 			      "vse8.v	v8, (%[wp2])\n"
528 			      "vse8.v	v9, (%[wq2])\n"
529 			      "vse8.v	v12, (%[wp3])\n"
530 			      "vse8.v	v13, (%[wq3])\n"
531 			      ".option	pop\n"
532 			      : :
533 			      [wp0]"r"(&p[d + nsize * 0]),
534 			      [wq0]"r"(&q[d + nsize * 0]),
535 			      [wp1]"r"(&p[d + nsize * 1]),
536 			      [wq1]"r"(&q[d + nsize * 1]),
537 			      [wp2]"r"(&p[d + nsize * 2]),
538 			      [wq2]"r"(&q[d + nsize * 2]),
539 			      [wp3]"r"(&p[d + nsize * 3]),
540 			      [wq3]"r"(&q[d + nsize * 3])
541 		);
542 	}
543 }
544 
545 static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
546 					 unsigned long bytes, void **ptrs)
547 {
548 	u8 **dptr = (u8 **)ptrs;
549 	u8 *p, *q;
550 	unsigned long vl, d, nsize;
551 	int z, z0;
552 
553 	z0 = stop;		/* P/Q right side optimization */
554 	p = dptr[disks - 2];	/* XOR parity */
555 	q = dptr[disks - 1];	/* RS syndrome */
556 
557 	asm volatile (".option	push\n"
558 		      ".option	arch,+v\n"
559 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
560 		      ".option	pop\n"
561 		      : "=&r" (vl)
562 	);
563 
564 	nsize = vl;
565 
566 	/*
567 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
568 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
569 	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
570 	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
571 	 */
572 	for (d = 0; d < bytes; d += nsize * 4) {
573 		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
574 		asm volatile (".option	push\n"
575 			      ".option	arch,+v\n"
576 			      "vle8.v	v0, (%[wp0])\n"
577 			      "vmv.v.v	v1, v0\n"
578 			      "vle8.v	v4, (%[wp1])\n"
579 			      "vmv.v.v	v5, v4\n"
580 			      "vle8.v	v8, (%[wp2])\n"
581 			      "vmv.v.v	v9, v8\n"
582 			      "vle8.v	v12, (%[wp3])\n"
583 			      "vmv.v.v	v13, v12\n"
584 			      ".option	pop\n"
585 			      : :
586 			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
587 			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
588 			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
589 			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
590 		);
591 
592 		/* P/Q data pages */
593 		for (z = z0 - 1; z >= start; z--) {
594 			/*
595 			 * w2$$ = MASK(wq$$);
596 			 * w1$$ = SHLBYTE(wq$$);
597 			 * w2$$ &= NBYTES(0x1d);
598 			 * w1$$ ^= w2$$;
599 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
600 			 * wq$$ = w1$$ ^ wd$$;
601 			 * wp$$ ^= wd$$;
602 			 */
603 			asm volatile (".option	push\n"
604 				      ".option	arch,+v\n"
605 				      "vsra.vi	v2, v1, 7\n"
606 				      "vsll.vi	v3, v1, 1\n"
607 				      "vand.vx	v2, v2, %[x1d]\n"
608 				      "vxor.vv	v3, v3, v2\n"
609 				      "vle8.v	v2, (%[wd0])\n"
610 				      "vxor.vv	v1, v3, v2\n"
611 				      "vxor.vv	v0, v0, v2\n"
612 
613 				      "vsra.vi	v6, v5, 7\n"
614 				      "vsll.vi	v7, v5, 1\n"
615 				      "vand.vx	v6, v6, %[x1d]\n"
616 				      "vxor.vv	v7, v7, v6\n"
617 				      "vle8.v	v6, (%[wd1])\n"
618 				      "vxor.vv	v5, v7, v6\n"
619 				      "vxor.vv	v4, v4, v6\n"
620 
621 				      "vsra.vi	v10, v9, 7\n"
622 				      "vsll.vi	v11, v9, 1\n"
623 				      "vand.vx	v10, v10, %[x1d]\n"
624 				      "vxor.vv	v11, v11, v10\n"
625 				      "vle8.v	v10, (%[wd2])\n"
626 				      "vxor.vv	v9, v11, v10\n"
627 				      "vxor.vv	v8, v8, v10\n"
628 
629 				      "vsra.vi	v14, v13, 7\n"
630 				      "vsll.vi	v15, v13, 1\n"
631 				      "vand.vx	v14, v14, %[x1d]\n"
632 				      "vxor.vv	v15, v15, v14\n"
633 				      "vle8.v	v14, (%[wd3])\n"
634 				      "vxor.vv	v13, v15, v14\n"
635 				      "vxor.vv	v12, v12, v14\n"
636 				      ".option	pop\n"
637 				      : :
638 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
639 				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
640 				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
641 				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
642 				      [x1d]"r"(0x1d)
643 			);
644 		}
645 
646 		/* P/Q left side optimization */
647 		for (z = start - 1; z >= 0; z--) {
648 			/*
649 			 * w2$$ = MASK(wq$$);
650 			 * w1$$ = SHLBYTE(wq$$);
651 			 * w2$$ &= NBYTES(0x1d);
652 			 * wq$$ = w1$$ ^ w2$$;
653 			 */
654 			asm volatile (".option	push\n"
655 				      ".option	arch,+v\n"
656 				      "vsra.vi	v2, v1, 7\n"
657 				      "vsll.vi	v3, v1, 1\n"
658 				      "vand.vx	v2, v2, %[x1d]\n"
659 				      "vxor.vv	v1, v3, v2\n"
660 
661 				      "vsra.vi	v6, v5, 7\n"
662 				      "vsll.vi	v7, v5, 1\n"
663 				      "vand.vx	v6, v6, %[x1d]\n"
664 				      "vxor.vv	v5, v7, v6\n"
665 
666 				      "vsra.vi	v10, v9, 7\n"
667 				      "vsll.vi	v11, v9, 1\n"
668 				      "vand.vx	v10, v10, %[x1d]\n"
669 				      "vxor.vv	v9, v11, v10\n"
670 
671 				      "vsra.vi	v14, v13, 7\n"
672 				      "vsll.vi	v15, v13, 1\n"
673 				      "vand.vx	v14, v14, %[x1d]\n"
674 				      "vxor.vv	v13, v15, v14\n"
675 				      ".option	pop\n"
676 				      : :
677 				      [x1d]"r"(0x1d)
678 			);
679 		}
680 
681 		/*
682 		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
683 		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
684 		 * v0:wp0, v1:wq0, v2:p0, v3:q0
685 		 * v4:wp1, v5:wq1, v6:p1, v7:q1
686 		 * v8:wp2, v9:wq2, v10:p2, v11:q2
687 		 * v12:wp3, v13:wq3, v14:p3, v15:q3
688 		 */
689 		asm volatile (".option	push\n"
690 			      ".option	arch,+v\n"
691 			      "vle8.v	v2, (%[wp0])\n"
692 			      "vle8.v	v3, (%[wq0])\n"
693 			      "vxor.vv	v2, v2, v0\n"
694 			      "vxor.vv	v3, v3, v1\n"
695 			      "vse8.v	v2, (%[wp0])\n"
696 			      "vse8.v	v3, (%[wq0])\n"
697 
698 			      "vle8.v	v6, (%[wp1])\n"
699 			      "vle8.v	v7, (%[wq1])\n"
700 			      "vxor.vv	v6, v6, v4\n"
701 			      "vxor.vv	v7, v7, v5\n"
702 			      "vse8.v	v6, (%[wp1])\n"
703 			      "vse8.v	v7, (%[wq1])\n"
704 
705 			      "vle8.v	v10, (%[wp2])\n"
706 			      "vle8.v	v11, (%[wq2])\n"
707 			      "vxor.vv	v10, v10, v8\n"
708 			      "vxor.vv	v11, v11, v9\n"
709 			      "vse8.v	v10, (%[wp2])\n"
710 			      "vse8.v	v11, (%[wq2])\n"
711 
712 			      "vle8.v	v14, (%[wp3])\n"
713 			      "vle8.v	v15, (%[wq3])\n"
714 			      "vxor.vv	v14, v14, v12\n"
715 			      "vxor.vv	v15, v15, v13\n"
716 			      "vse8.v	v14, (%[wp3])\n"
717 			      "vse8.v	v15, (%[wq3])\n"
718 			      ".option	pop\n"
719 			      : :
720 			      [wp0]"r"(&p[d + nsize * 0]),
721 			      [wq0]"r"(&q[d + nsize * 0]),
722 			      [wp1]"r"(&p[d + nsize * 1]),
723 			      [wq1]"r"(&q[d + nsize * 1]),
724 			      [wp2]"r"(&p[d + nsize * 2]),
725 			      [wq2]"r"(&q[d + nsize * 2]),
726 			      [wp3]"r"(&p[d + nsize * 3]),
727 			      [wq3]"r"(&q[d + nsize * 3])
728 		);
729 	}
730 }
731 
732 static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
733 {
734 	u8 **dptr = (u8 **)ptrs;
735 	u8 *p, *q;
736 	unsigned long vl, d, nsize;
737 	int z, z0;
738 
739 	z0 = disks - 3;	/* Highest data disk */
740 	p = dptr[z0 + 1];	/* XOR parity */
741 	q = dptr[z0 + 2];	/* RS syndrome */
742 
743 	asm volatile (".option	push\n"
744 		      ".option	arch,+v\n"
745 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
746 		      ".option	pop\n"
747 		      : "=&r" (vl)
748 	);
749 
750 	nsize = vl;
751 
752 	/*
753 	 * v0:wp0,   v1:wq0,  v2:wd0/w20,  v3:w10
754 	 * v4:wp1,   v5:wq1,  v6:wd1/w21,  v7:w11
755 	 * v8:wp2,   v9:wq2, v10:wd2/w22, v11:w12
756 	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
757 	 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
758 	 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
759 	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
760 	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
761 	 */
762 	for (d = 0; d < bytes; d += nsize * 8) {
763 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
764 		asm volatile (".option	push\n"
765 			      ".option	arch,+v\n"
766 			      "vle8.v	v0, (%[wp0])\n"
767 			      "vmv.v.v	v1, v0\n"
768 			      "vle8.v	v4, (%[wp1])\n"
769 			      "vmv.v.v	v5, v4\n"
770 			      "vle8.v	v8, (%[wp2])\n"
771 			      "vmv.v.v	v9, v8\n"
772 			      "vle8.v	v12, (%[wp3])\n"
773 			      "vmv.v.v	v13, v12\n"
774 			      "vle8.v	v16, (%[wp4])\n"
775 			      "vmv.v.v	v17, v16\n"
776 			      "vle8.v	v20, (%[wp5])\n"
777 			      "vmv.v.v	v21, v20\n"
778 			      "vle8.v	v24, (%[wp6])\n"
779 			      "vmv.v.v	v25, v24\n"
780 			      "vle8.v	v28, (%[wp7])\n"
781 			      "vmv.v.v	v29, v28\n"
782 			      ".option	pop\n"
783 			      : :
784 			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
785 			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
786 			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
787 			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
788 			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
789 			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
790 			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
791 			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
792 		);
793 
794 		for (z = z0 - 1; z >= 0; z--) {
795 			/*
796 			 * w2$$ = MASK(wq$$);
797 			 * w1$$ = SHLBYTE(wq$$);
798 			 * w2$$ &= NBYTES(0x1d);
799 			 * w1$$ ^= w2$$;
800 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
801 			 * wq$$ = w1$$ ^ wd$$;
802 			 * wp$$ ^= wd$$;
803 			 */
804 			asm volatile (".option	push\n"
805 				      ".option	arch,+v\n"
806 				      "vsra.vi	v2, v1, 7\n"
807 				      "vsll.vi	v3, v1, 1\n"
808 				      "vand.vx	v2, v2, %[x1d]\n"
809 				      "vxor.vv	v3, v3, v2\n"
810 				      "vle8.v	v2, (%[wd0])\n"
811 				      "vxor.vv	v1, v3, v2\n"
812 				      "vxor.vv	v0, v0, v2\n"
813 
814 				      "vsra.vi	v6, v5, 7\n"
815 				      "vsll.vi	v7, v5, 1\n"
816 				      "vand.vx	v6, v6, %[x1d]\n"
817 				      "vxor.vv	v7, v7, v6\n"
818 				      "vle8.v	v6, (%[wd1])\n"
819 				      "vxor.vv	v5, v7, v6\n"
820 				      "vxor.vv	v4, v4, v6\n"
821 
822 				      "vsra.vi	v10, v9, 7\n"
823 				      "vsll.vi	v11, v9, 1\n"
824 				      "vand.vx	v10, v10, %[x1d]\n"
825 				      "vxor.vv	v11, v11, v10\n"
826 				      "vle8.v	v10, (%[wd2])\n"
827 				      "vxor.vv	v9, v11, v10\n"
828 				      "vxor.vv	v8, v8, v10\n"
829 
830 				      "vsra.vi	v14, v13, 7\n"
831 				      "vsll.vi	v15, v13, 1\n"
832 				      "vand.vx	v14, v14, %[x1d]\n"
833 				      "vxor.vv	v15, v15, v14\n"
834 				      "vle8.v	v14, (%[wd3])\n"
835 				      "vxor.vv	v13, v15, v14\n"
836 				      "vxor.vv	v12, v12, v14\n"
837 
838 				      "vsra.vi	v18, v17, 7\n"
839 				      "vsll.vi	v19, v17, 1\n"
840 				      "vand.vx	v18, v18, %[x1d]\n"
841 				      "vxor.vv	v19, v19, v18\n"
842 				      "vle8.v	v18, (%[wd4])\n"
843 				      "vxor.vv	v17, v19, v18\n"
844 				      "vxor.vv	v16, v16, v18\n"
845 
846 				      "vsra.vi	v22, v21, 7\n"
847 				      "vsll.vi	v23, v21, 1\n"
848 				      "vand.vx	v22, v22, %[x1d]\n"
849 				      "vxor.vv	v23, v23, v22\n"
850 				      "vle8.v	v22, (%[wd5])\n"
851 				      "vxor.vv	v21, v23, v22\n"
852 				      "vxor.vv	v20, v20, v22\n"
853 
854 				      "vsra.vi	v26, v25, 7\n"
855 				      "vsll.vi	v27, v25, 1\n"
856 				      "vand.vx	v26, v26, %[x1d]\n"
857 				      "vxor.vv	v27, v27, v26\n"
858 				      "vle8.v	v26, (%[wd6])\n"
859 				      "vxor.vv	v25, v27, v26\n"
860 				      "vxor.vv	v24, v24, v26\n"
861 
862 				      "vsra.vi	v30, v29, 7\n"
863 				      "vsll.vi	v31, v29, 1\n"
864 				      "vand.vx	v30, v30, %[x1d]\n"
865 				      "vxor.vv	v31, v31, v30\n"
866 				      "vle8.v	v30, (%[wd7])\n"
867 				      "vxor.vv	v29, v31, v30\n"
868 				      "vxor.vv	v28, v28, v30\n"
869 				      ".option	pop\n"
870 				      : :
871 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
872 				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
873 				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
874 				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
875 				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
876 				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
877 				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
878 				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
879 				      [x1d]"r"(0x1d)
880 			);
881 		}
882 
883 		/*
884 		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
885 		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
886 		 */
887 		asm volatile (".option	push\n"
888 			      ".option	arch,+v\n"
889 			      "vse8.v	v0, (%[wp0])\n"
890 			      "vse8.v	v1, (%[wq0])\n"
891 			      "vse8.v	v4, (%[wp1])\n"
892 			      "vse8.v	v5, (%[wq1])\n"
893 			      "vse8.v	v8, (%[wp2])\n"
894 			      "vse8.v	v9, (%[wq2])\n"
895 			      "vse8.v	v12, (%[wp3])\n"
896 			      "vse8.v	v13, (%[wq3])\n"
897 			      "vse8.v	v16, (%[wp4])\n"
898 			      "vse8.v	v17, (%[wq4])\n"
899 			      "vse8.v	v20, (%[wp5])\n"
900 			      "vse8.v	v21, (%[wq5])\n"
901 			      "vse8.v	v24, (%[wp6])\n"
902 			      "vse8.v	v25, (%[wq6])\n"
903 			      "vse8.v	v28, (%[wp7])\n"
904 			      "vse8.v	v29, (%[wq7])\n"
905 			      ".option	pop\n"
906 			      : :
907 			      [wp0]"r"(&p[d + nsize * 0]),
908 			      [wq0]"r"(&q[d + nsize * 0]),
909 			      [wp1]"r"(&p[d + nsize * 1]),
910 			      [wq1]"r"(&q[d + nsize * 1]),
911 			      [wp2]"r"(&p[d + nsize * 2]),
912 			      [wq2]"r"(&q[d + nsize * 2]),
913 			      [wp3]"r"(&p[d + nsize * 3]),
914 			      [wq3]"r"(&q[d + nsize * 3]),
915 			      [wp4]"r"(&p[d + nsize * 4]),
916 			      [wq4]"r"(&q[d + nsize * 4]),
917 			      [wp5]"r"(&p[d + nsize * 5]),
918 			      [wq5]"r"(&q[d + nsize * 5]),
919 			      [wp6]"r"(&p[d + nsize * 6]),
920 			      [wq6]"r"(&q[d + nsize * 6]),
921 			      [wp7]"r"(&p[d + nsize * 7]),
922 			      [wq7]"r"(&q[d + nsize * 7])
923 		);
924 	}
925 }
926 
927 static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
928 					 unsigned long bytes, void **ptrs)
929 {
930 	u8 **dptr = (u8 **)ptrs;
931 	u8 *p, *q;
932 	unsigned long vl, d, nsize;
933 	int z, z0;
934 
935 	z0 = stop;		/* P/Q right side optimization */
936 	p = dptr[disks - 2];	/* XOR parity */
937 	q = dptr[disks - 1];	/* RS syndrome */
938 
939 	asm volatile (".option	push\n"
940 		      ".option	arch,+v\n"
941 		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
942 		      ".option	pop\n"
943 		      : "=&r" (vl)
944 	);
945 
946 	nsize = vl;
947 
948 	/*
949 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
950 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
951 	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
952 	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
953 	 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
954 	 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
955 	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
956 	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
957 	 */
958 	for (d = 0; d < bytes; d += nsize * 8) {
959 		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
960 		asm volatile (".option	push\n"
961 			      ".option	arch,+v\n"
962 			      "vle8.v	v0, (%[wp0])\n"
963 			      "vmv.v.v	v1, v0\n"
964 			      "vle8.v	v4, (%[wp1])\n"
965 			      "vmv.v.v	v5, v4\n"
966 			      "vle8.v	v8, (%[wp2])\n"
967 			      "vmv.v.v	v9, v8\n"
968 			      "vle8.v	v12, (%[wp3])\n"
969 			      "vmv.v.v	v13, v12\n"
970 			      "vle8.v	v16, (%[wp4])\n"
971 			      "vmv.v.v	v17, v16\n"
972 			      "vle8.v	v20, (%[wp5])\n"
973 			      "vmv.v.v	v21, v20\n"
974 			      "vle8.v	v24, (%[wp6])\n"
975 			      "vmv.v.v	v25, v24\n"
976 			      "vle8.v	v28, (%[wp7])\n"
977 			      "vmv.v.v	v29, v28\n"
978 			      ".option	pop\n"
979 			      : :
980 			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
981 			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
982 			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
983 			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
984 			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
985 			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
986 			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
987 			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
988 		);
989 
990 		/* P/Q data pages */
991 		for (z = z0 - 1; z >= start; z--) {
992 			/*
993 			 * w2$$ = MASK(wq$$);
994 			 * w1$$ = SHLBYTE(wq$$);
995 			 * w2$$ &= NBYTES(0x1d);
996 			 * w1$$ ^= w2$$;
997 			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
998 			 * wq$$ = w1$$ ^ wd$$;
999 			 * wp$$ ^= wd$$;
1000 			 */
1001 			asm volatile (".option	push\n"
1002 				      ".option	arch,+v\n"
1003 				      "vsra.vi	v2, v1, 7\n"
1004 				      "vsll.vi	v3, v1, 1\n"
1005 				      "vand.vx	v2, v2, %[x1d]\n"
1006 				      "vxor.vv	v3, v3, v2\n"
1007 				      "vle8.v	v2, (%[wd0])\n"
1008 				      "vxor.vv	v1, v3, v2\n"
1009 				      "vxor.vv	v0, v0, v2\n"
1010 
1011 				      "vsra.vi	v6, v5, 7\n"
1012 				      "vsll.vi	v7, v5, 1\n"
1013 				      "vand.vx	v6, v6, %[x1d]\n"
1014 				      "vxor.vv	v7, v7, v6\n"
1015 				      "vle8.v	v6, (%[wd1])\n"
1016 				      "vxor.vv	v5, v7, v6\n"
1017 				      "vxor.vv	v4, v4, v6\n"
1018 
1019 				      "vsra.vi	v10, v9, 7\n"
1020 				      "vsll.vi	v11, v9, 1\n"
1021 				      "vand.vx	v10, v10, %[x1d]\n"
1022 				      "vxor.vv	v11, v11, v10\n"
1023 				      "vle8.v	v10, (%[wd2])\n"
1024 				      "vxor.vv	v9, v11, v10\n"
1025 				      "vxor.vv	v8, v8, v10\n"
1026 
1027 				      "vsra.vi	v14, v13, 7\n"
1028 				      "vsll.vi	v15, v13, 1\n"
1029 				      "vand.vx	v14, v14, %[x1d]\n"
1030 				      "vxor.vv	v15, v15, v14\n"
1031 				      "vle8.v	v14, (%[wd3])\n"
1032 				      "vxor.vv	v13, v15, v14\n"
1033 				      "vxor.vv	v12, v12, v14\n"
1034 
1035 				      "vsra.vi	v18, v17, 7\n"
1036 				      "vsll.vi	v19, v17, 1\n"
1037 				      "vand.vx	v18, v18, %[x1d]\n"
1038 				      "vxor.vv	v19, v19, v18\n"
1039 				      "vle8.v	v18, (%[wd4])\n"
1040 				      "vxor.vv	v17, v19, v18\n"
1041 				      "vxor.vv	v16, v16, v18\n"
1042 
1043 				      "vsra.vi	v22, v21, 7\n"
1044 				      "vsll.vi	v23, v21, 1\n"
1045 				      "vand.vx	v22, v22, %[x1d]\n"
1046 				      "vxor.vv	v23, v23, v22\n"
1047 				      "vle8.v	v22, (%[wd5])\n"
1048 				      "vxor.vv	v21, v23, v22\n"
1049 				      "vxor.vv	v20, v20, v22\n"
1050 
1051 				      "vsra.vi	v26, v25, 7\n"
1052 				      "vsll.vi	v27, v25, 1\n"
1053 				      "vand.vx	v26, v26, %[x1d]\n"
1054 				      "vxor.vv	v27, v27, v26\n"
1055 				      "vle8.v	v26, (%[wd6])\n"
1056 				      "vxor.vv	v25, v27, v26\n"
1057 				      "vxor.vv	v24, v24, v26\n"
1058 
1059 				      "vsra.vi	v30, v29, 7\n"
1060 				      "vsll.vi	v31, v29, 1\n"
1061 				      "vand.vx	v30, v30, %[x1d]\n"
1062 				      "vxor.vv	v31, v31, v30\n"
1063 				      "vle8.v	v30, (%[wd7])\n"
1064 				      "vxor.vv	v29, v31, v30\n"
1065 				      "vxor.vv	v28, v28, v30\n"
1066 				      ".option	pop\n"
1067 				      : :
1068 				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
1069 				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
1070 				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
1071 				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
1072 				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
1073 				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
1074 				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
1075 				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
1076 				      [x1d]"r"(0x1d)
1077 			);
1078 		}
1079 
1080 		/* P/Q left side optimization */
1081 		for (z = start - 1; z >= 0; z--) {
1082 			/*
1083 			 * w2$$ = MASK(wq$$);
1084 			 * w1$$ = SHLBYTE(wq$$);
1085 			 * w2$$ &= NBYTES(0x1d);
1086 			 * wq$$ = w1$$ ^ w2$$;
1087 			 */
1088 			asm volatile (".option	push\n"
1089 				      ".option	arch,+v\n"
1090 				      "vsra.vi	v2, v1, 7\n"
1091 				      "vsll.vi	v3, v1, 1\n"
1092 				      "vand.vx	v2, v2, %[x1d]\n"
1093 				      "vxor.vv	v1, v3, v2\n"
1094 
1095 				      "vsra.vi	v6, v5, 7\n"
1096 				      "vsll.vi	v7, v5, 1\n"
1097 				      "vand.vx	v6, v6, %[x1d]\n"
1098 				      "vxor.vv	v5, v7, v6\n"
1099 
1100 				      "vsra.vi	v10, v9, 7\n"
1101 				      "vsll.vi	v11, v9, 1\n"
1102 				      "vand.vx	v10, v10, %[x1d]\n"
1103 				      "vxor.vv	v9, v11, v10\n"
1104 
1105 				      "vsra.vi	v14, v13, 7\n"
1106 				      "vsll.vi	v15, v13, 1\n"
1107 				      "vand.vx	v14, v14, %[x1d]\n"
1108 				      "vxor.vv	v13, v15, v14\n"
1109 
1110 				      "vsra.vi	v18, v17, 7\n"
1111 				      "vsll.vi	v19, v17, 1\n"
1112 				      "vand.vx	v18, v18, %[x1d]\n"
1113 				      "vxor.vv	v17, v19, v18\n"
1114 
1115 				      "vsra.vi	v22, v21, 7\n"
1116 				      "vsll.vi	v23, v21, 1\n"
1117 				      "vand.vx	v22, v22, %[x1d]\n"
1118 				      "vxor.vv	v21, v23, v22\n"
1119 
1120 				      "vsra.vi	v26, v25, 7\n"
1121 				      "vsll.vi	v27, v25, 1\n"
1122 				      "vand.vx	v26, v26, %[x1d]\n"
1123 				      "vxor.vv	v25, v27, v26\n"
1124 
1125 				      "vsra.vi	v30, v29, 7\n"
1126 				      "vsll.vi	v31, v29, 1\n"
1127 				      "vand.vx	v30, v30, %[x1d]\n"
1128 				      "vxor.vv	v29, v31, v30\n"
1129 				      ".option	pop\n"
1130 				      : :
1131 				      [x1d]"r"(0x1d)
1132 			);
1133 		}
1134 
1135 		/*
1136 		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
1137 		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
1138 		 * v0:wp0, v1:wq0, v2:p0, v3:q0
1139 		 * v4:wp1, v5:wq1, v6:p1, v7:q1
1140 		 * v8:wp2, v9:wq2, v10:p2, v11:q2
1141 		 * v12:wp3, v13:wq3, v14:p3, v15:q3
1142 		 * v16:wp4, v17:wq4, v18:p4, v19:q4
1143 		 * v20:wp5, v21:wq5, v22:p5, v23:q5
1144 		 * v24:wp6, v25:wq6, v26:p6, v27:q6
1145 		 * v28:wp7, v29:wq7, v30:p7, v31:q7
1146 		 */
1147 		asm volatile (".option	push\n"
1148 			      ".option	arch,+v\n"
1149 			      "vle8.v	v2, (%[wp0])\n"
1150 			      "vle8.v	v3, (%[wq0])\n"
1151 			      "vxor.vv	v2, v2, v0\n"
1152 			      "vxor.vv	v3, v3, v1\n"
1153 			      "vse8.v	v2, (%[wp0])\n"
1154 			      "vse8.v	v3, (%[wq0])\n"
1155 
1156 			      "vle8.v	v6, (%[wp1])\n"
1157 			      "vle8.v	v7, (%[wq1])\n"
1158 			      "vxor.vv	v6, v6, v4\n"
1159 			      "vxor.vv	v7, v7, v5\n"
1160 			      "vse8.v	v6, (%[wp1])\n"
1161 			      "vse8.v	v7, (%[wq1])\n"
1162 
1163 			      "vle8.v	v10, (%[wp2])\n"
1164 			      "vle8.v	v11, (%[wq2])\n"
1165 			      "vxor.vv	v10, v10, v8\n"
1166 			      "vxor.vv	v11, v11, v9\n"
1167 			      "vse8.v	v10, (%[wp2])\n"
1168 			      "vse8.v	v11, (%[wq2])\n"
1169 
1170 			      "vle8.v	v14, (%[wp3])\n"
1171 			      "vle8.v	v15, (%[wq3])\n"
1172 			      "vxor.vv	v14, v14, v12\n"
1173 			      "vxor.vv	v15, v15, v13\n"
1174 			      "vse8.v	v14, (%[wp3])\n"
1175 			      "vse8.v	v15, (%[wq3])\n"
1176 
1177 			      "vle8.v	v18, (%[wp4])\n"
1178 			      "vle8.v	v19, (%[wq4])\n"
1179 			      "vxor.vv	v18, v18, v16\n"
1180 			      "vxor.vv	v19, v19, v17\n"
1181 			      "vse8.v	v18, (%[wp4])\n"
1182 			      "vse8.v	v19, (%[wq4])\n"
1183 
1184 			      "vle8.v	v22, (%[wp5])\n"
1185 			      "vle8.v	v23, (%[wq5])\n"
1186 			      "vxor.vv	v22, v22, v20\n"
1187 			      "vxor.vv	v23, v23, v21\n"
1188 			      "vse8.v	v22, (%[wp5])\n"
1189 			      "vse8.v	v23, (%[wq5])\n"
1190 
1191 			      "vle8.v	v26, (%[wp6])\n"
1192 			      "vle8.v	v27, (%[wq6])\n"
1193 			      "vxor.vv	v26, v26, v24\n"
1194 			      "vxor.vv	v27, v27, v25\n"
1195 			      "vse8.v	v26, (%[wp6])\n"
1196 			      "vse8.v	v27, (%[wq6])\n"
1197 
1198 			      "vle8.v	v30, (%[wp7])\n"
1199 			      "vle8.v	v31, (%[wq7])\n"
1200 			      "vxor.vv	v30, v30, v28\n"
1201 			      "vxor.vv	v31, v31, v29\n"
1202 			      "vse8.v	v30, (%[wp7])\n"
1203 			      "vse8.v	v31, (%[wq7])\n"
1204 			      ".option	pop\n"
1205 			      : :
1206 			      [wp0]"r"(&p[d + nsize * 0]),
1207 			      [wq0]"r"(&q[d + nsize * 0]),
1208 			      [wp1]"r"(&p[d + nsize * 1]),
1209 			      [wq1]"r"(&q[d + nsize * 1]),
1210 			      [wp2]"r"(&p[d + nsize * 2]),
1211 			      [wq2]"r"(&q[d + nsize * 2]),
1212 			      [wp3]"r"(&p[d + nsize * 3]),
1213 			      [wq3]"r"(&q[d + nsize * 3]),
1214 			      [wp4]"r"(&p[d + nsize * 4]),
1215 			      [wq4]"r"(&q[d + nsize * 4]),
1216 			      [wp5]"r"(&p[d + nsize * 5]),
1217 			      [wq5]"r"(&q[d + nsize * 5]),
1218 			      [wp6]"r"(&p[d + nsize * 6]),
1219 			      [wq6]"r"(&q[d + nsize * 6]),
1220 			      [wp7]"r"(&p[d + nsize * 7]),
1221 			      [wq7]"r"(&q[d + nsize * 7])
1222 		);
1223 	}
1224 }
1225 
1226 RAID6_RVV_WRAPPER(1);
1227 RAID6_RVV_WRAPPER(2);
1228 RAID6_RVV_WRAPPER(4);
1229 RAID6_RVV_WRAPPER(8);
1230