xref: /linux/lib/raid6/avx512.c (revision 0fdebc5ec2ca492d69df2d93a6a7abade4941aae) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* -*- linux-c -*- --------------------------------------------------------
3  *
4  *   Copyright (C) 2016 Intel Corporation
5  *
6  *   Author: Gayatri Kammela <gayatri.kammela@intel.com>
7  *   Author: Megha Dey <megha.dey@linux.intel.com>
8  *
9  *   Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
10  *   Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
11  *
12  * -----------------------------------------------------------------------
13  */
14 
15 /*
16  * AVX512 implementation of RAID-6 syndrome functions
17  *
18  */
19 
20 #ifdef CONFIG_AS_AVX512
21 
22 #include <linux/raid/pq.h>
23 #include "x86.h"
24 
25 static const struct raid6_avx512_constants {
26 	u64 x1d[8];
27 } raid6_avx512_constants __aligned(512/8) = {
28 	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
29 	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
30 	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
31 	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
32 };
33 
34 static int raid6_have_avx512(void)
35 {
36 	return boot_cpu_has(X86_FEATURE_AVX2) &&
37 		boot_cpu_has(X86_FEATURE_AVX) &&
38 		boot_cpu_has(X86_FEATURE_AVX512F) &&
39 		boot_cpu_has(X86_FEATURE_AVX512BW) &&
40 		boot_cpu_has(X86_FEATURE_AVX512VL) &&
41 		boot_cpu_has(X86_FEATURE_AVX512DQ);
42 }
43 
44 static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
45 {
46 	u8 **dptr = (u8 **)ptrs;
47 	u8 *p, *q;
48 	int d, z, z0;
49 
50 	z0 = disks - 3;         /* Highest data disk */
51 	p = dptr[z0+1];         /* XOR parity */
52 	q = dptr[z0+2];         /* RS syndrome */
53 
54 	kernel_fpu_begin();
55 
56 	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
57 		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
58 		     :
59 		     : "m" (raid6_avx512_constants.x1d[0]));
60 
61 	for (d = 0; d < bytes; d += 64) {
62 		asm volatile("prefetchnta %0\n\t"
63 			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
64 			     "prefetchnta %1\n\t"
65 			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
66 			     "vmovdqa64 %1,%%zmm6"
67 			     :
68 			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
69 		for (z = z0-2; z >= 0; z--) {
70 			asm volatile("prefetchnta %0\n\t"
71 				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
72 				     "vpmovm2b %%k1,%%zmm5\n\t"
73 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
74 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
75 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
76 				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
77 				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
78 				     "vmovdqa64 %0,%%zmm6"
79 				     :
80 				     : "m" (dptr[z][d]));
81 		}
82 		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
83 			     "vpmovm2b %%k1,%%zmm5\n\t"
84 			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
85 			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
86 			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
87 			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
88 			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
89 			     "vmovntdq %%zmm2,%0\n\t"
90 			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
91 			     "vmovntdq %%zmm4,%1\n\t"
92 			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
93 			     :
94 			     : "m" (p[d]), "m" (q[d]));
95 	}
96 
97 	asm volatile("sfence" : : : "memory");
98 	kernel_fpu_end();
99 }
100 
101 static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
102 				       size_t bytes, void **ptrs)
103 {
104 	u8 **dptr = (u8 **)ptrs;
105 	u8 *p, *q;
106 	int d, z, z0;
107 
108 	z0 = stop;		/* P/Q right side optimization */
109 	p = dptr[disks-2];	/* XOR parity */
110 	q = dptr[disks-1];	/* RS syndrome */
111 
112 	kernel_fpu_begin();
113 
114 	asm volatile("vmovdqa64 %0,%%zmm0"
115 		     : : "m" (raid6_avx512_constants.x1d[0]));
116 
117 	for (d = 0 ; d < bytes ; d += 64) {
118 		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
119 			     "vmovdqa64 %1,%%zmm2\n\t"
120 			     "vpxorq %%zmm4,%%zmm2,%%zmm2"
121 			     :
122 			     : "m" (dptr[z0][d]),  "m" (p[d]));
123 		/* P/Q data pages */
124 		for (z = z0-1 ; z >= start ; z--) {
125 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
126 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
127 				     "vpmovm2b %%k1,%%zmm5\n\t"
128 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
129 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
130 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
131 				     "vmovdqa64 %0,%%zmm5\n\t"
132 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
133 				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
134 				     :
135 				     : "m" (dptr[z][d]));
136 		}
137 		/* P/Q left side optimization */
138 		for (z = start-1 ; z >= 0 ; z--) {
139 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
140 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
141 				     "vpmovm2b %%k1,%%zmm5\n\t"
142 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
143 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
144 				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
145 				     :
146 				     : );
147 		}
148 		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
149 		/* Don't use movntdq for r/w memory area < cache line */
150 			     "vmovdqa64 %%zmm4,%0\n\t"
151 			     "vmovdqa64 %%zmm2,%1"
152 			     :
153 			     : "m" (q[d]), "m" (p[d]));
154 	}
155 
156 	asm volatile("sfence" : : : "memory");
157 	kernel_fpu_end();
158 }
159 
160 const struct raid6_calls raid6_avx512x1 = {
161 	raid6_avx5121_gen_syndrome,
162 	raid6_avx5121_xor_syndrome,
163 	raid6_have_avx512,
164 	"avx512x1",
165 	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
166 };
167 
168 /*
169  * Unrolled-by-2 AVX512 implementation
170  */
171 static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
172 {
173 	u8 **dptr = (u8 **)ptrs;
174 	u8 *p, *q;
175 	int d, z, z0;
176 
177 	z0 = disks - 3;         /* Highest data disk */
178 	p = dptr[z0+1];         /* XOR parity */
179 	q = dptr[z0+2];         /* RS syndrome */
180 
181 	kernel_fpu_begin();
182 
183 	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
184 		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
185 		     :
186 		     : "m" (raid6_avx512_constants.x1d[0]));
187 
188 	/* We uniformly assume a single prefetch covers at least 64 bytes */
189 	for (d = 0; d < bytes; d += 128) {
190 		asm volatile("prefetchnta %0\n\t"
191 			     "prefetchnta %1\n\t"
192 			     "vmovdqa64 %0,%%zmm2\n\t"      /* P[0] */
193 			     "vmovdqa64 %1,%%zmm3\n\t"      /* P[1] */
194 			     "vmovdqa64 %%zmm2,%%zmm4\n\t"  /* Q[0] */
195 			     "vmovdqa64 %%zmm3,%%zmm6"      /* Q[1] */
196 			     :
197 			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
198 		for (z = z0-1; z >= 0; z--) {
199 			asm volatile("prefetchnta %0\n\t"
200 				     "prefetchnta %1\n\t"
201 				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
202 				     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
203 				     "vpmovm2b %%k1,%%zmm5\n\t"
204 				     "vpmovm2b %%k2,%%zmm7\n\t"
205 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
206 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
207 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
208 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
209 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
210 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
211 				     "vmovdqa64 %0,%%zmm5\n\t"
212 				     "vmovdqa64 %1,%%zmm7\n\t"
213 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
214 				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
215 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
216 				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
217 				     :
218 				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
219 		}
220 		asm volatile("vmovntdq %%zmm2,%0\n\t"
221 			     "vmovntdq %%zmm3,%1\n\t"
222 			     "vmovntdq %%zmm4,%2\n\t"
223 			     "vmovntdq %%zmm6,%3"
224 			     :
225 			     : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
226 			       "m" (q[d+64]));
227 	}
228 
229 	asm volatile("sfence" : : : "memory");
230 	kernel_fpu_end();
231 }
232 
233 static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
234 				       size_t bytes, void **ptrs)
235 {
236 	u8 **dptr = (u8 **)ptrs;
237 	u8 *p, *q;
238 	int d, z, z0;
239 
240 	z0 = stop;		/* P/Q right side optimization */
241 	p = dptr[disks-2];	/* XOR parity */
242 	q = dptr[disks-1];	/* RS syndrome */
243 
244 	kernel_fpu_begin();
245 
246 	asm volatile("vmovdqa64 %0,%%zmm0"
247 		     : : "m" (raid6_avx512_constants.x1d[0]));
248 
249 	for (d = 0 ; d < bytes ; d += 128) {
250 		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
251 			     "vmovdqa64 %1,%%zmm6\n\t"
252 			     "vmovdqa64 %2,%%zmm2\n\t"
253 			     "vmovdqa64 %3,%%zmm3\n\t"
254 			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
255 			     "vpxorq %%zmm6,%%zmm3,%%zmm3"
256 			     :
257 			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
258 			       "m" (p[d]), "m" (p[d+64]));
259 		/* P/Q data pages */
260 		for (z = z0-1 ; z >= start ; z--) {
261 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
262 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
263 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
264 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
265 				     "vpmovm2b %%k1,%%zmm5\n\t"
266 				     "vpmovm2b %%k2,%%zmm7\n\t"
267 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
268 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
269 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
270 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
271 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
272 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
273 				     "vmovdqa64 %0,%%zmm5\n\t"
274 				     "vmovdqa64 %1,%%zmm7\n\t"
275 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
276 				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
277 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
278 				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
279 				     :
280 				     : "m" (dptr[z][d]),  "m" (dptr[z][d+64]));
281 		}
282 		/* P/Q left side optimization */
283 		for (z = start-1 ; z >= 0 ; z--) {
284 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
285 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
286 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
287 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
288 				     "vpmovm2b %%k1,%%zmm5\n\t"
289 				     "vpmovm2b %%k2,%%zmm7\n\t"
290 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
291 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
292 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
293 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
294 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
295 				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
296 				     :
297 				     : );
298 		}
299 		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
300 			     "vpxorq %1,%%zmm6,%%zmm6\n\t"
301 			     /* Don't use movntdq for r/w
302 			      * memory area < cache line
303 			      */
304 			     "vmovdqa64 %%zmm4,%0\n\t"
305 			     "vmovdqa64 %%zmm6,%1\n\t"
306 			     "vmovdqa64 %%zmm2,%2\n\t"
307 			     "vmovdqa64 %%zmm3,%3"
308 			     :
309 			     : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
310 			       "m" (p[d+64]));
311 	}
312 
313 	asm volatile("sfence" : : : "memory");
314 	kernel_fpu_end();
315 }
316 
317 const struct raid6_calls raid6_avx512x2 = {
318 	raid6_avx5122_gen_syndrome,
319 	raid6_avx5122_xor_syndrome,
320 	raid6_have_avx512,
321 	"avx512x2",
322 	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
323 };
324 
325 #ifdef CONFIG_X86_64
326 
327 /*
328  * Unrolled-by-4 AVX2 implementation
329  */
330 static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
331 {
332 	u8 **dptr = (u8 **)ptrs;
333 	u8 *p, *q;
334 	int d, z, z0;
335 
336 	z0 = disks - 3;         /* Highest data disk */
337 	p = dptr[z0+1];         /* XOR parity */
338 	q = dptr[z0+2];         /* RS syndrome */
339 
340 	kernel_fpu_begin();
341 
342 	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
343 		     "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t"       /* Zero temp */
344 		     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"       /* P[0] */
345 		     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"       /* P[1] */
346 		     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"       /* Q[0] */
347 		     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"       /* Q[1] */
348 		     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"    /* P[2] */
349 		     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"    /* P[3] */
350 		     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"    /* Q[2] */
351 		     "vpxorq %%zmm14,%%zmm14,%%zmm14"        /* Q[3] */
352 		     :
353 		     : "m" (raid6_avx512_constants.x1d[0]));
354 
355 	for (d = 0; d < bytes; d += 256) {
356 		for (z = z0; z >= 0; z--) {
357 		asm volatile("prefetchnta %0\n\t"
358 			     "prefetchnta %1\n\t"
359 			     "prefetchnta %2\n\t"
360 			     "prefetchnta %3\n\t"
361 			     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
362 			     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
363 			     "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
364 			     "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
365 			     "vpmovm2b %%k1,%%zmm5\n\t"
366 			     "vpmovm2b %%k2,%%zmm7\n\t"
367 			     "vpmovm2b %%k3,%%zmm13\n\t"
368 			     "vpmovm2b %%k4,%%zmm15\n\t"
369 			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
370 			     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
371 			     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
372 			     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
373 			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
374 			     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
375 			     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
376 			     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
377 			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
378 			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
379 			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
380 			     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
381 			     "vmovdqa64 %0,%%zmm5\n\t"
382 			     "vmovdqa64 %1,%%zmm7\n\t"
383 			     "vmovdqa64 %2,%%zmm13\n\t"
384 			     "vmovdqa64 %3,%%zmm15\n\t"
385 			     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
386 			     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
387 			     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
388 			     "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
389 			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
390 			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
391 			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
392 			     "vpxorq %%zmm15,%%zmm14,%%zmm14"
393 			     :
394 			     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
395 			       "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
396 		}
397 		asm volatile("vmovntdq %%zmm2,%0\n\t"
398 			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
399 			     "vmovntdq %%zmm3,%1\n\t"
400 			     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
401 			     "vmovntdq %%zmm10,%2\n\t"
402 			     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
403 			     "vmovntdq %%zmm11,%3\n\t"
404 			     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
405 			     "vmovntdq %%zmm4,%4\n\t"
406 			     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
407 			     "vmovntdq %%zmm6,%5\n\t"
408 			     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
409 			     "vmovntdq %%zmm12,%6\n\t"
410 			     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
411 			     "vmovntdq %%zmm14,%7\n\t"
412 			     "vpxorq %%zmm14,%%zmm14,%%zmm14"
413 			     :
414 			     : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
415 			       "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
416 			       "m" (q[d+128]), "m" (q[d+192]));
417 	}
418 
419 	asm volatile("sfence" : : : "memory");
420 	kernel_fpu_end();
421 }
422 
423 static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
424 				       size_t bytes, void **ptrs)
425 {
426 	u8 **dptr = (u8 **)ptrs;
427 	u8 *p, *q;
428 	int d, z, z0;
429 
430 	z0 = stop;		/* P/Q right side optimization */
431 	p = dptr[disks-2];	/* XOR parity */
432 	q = dptr[disks-1];	/* RS syndrome */
433 
434 	kernel_fpu_begin();
435 
436 	asm volatile("vmovdqa64 %0,%%zmm0"
437 		     :: "m" (raid6_avx512_constants.x1d[0]));
438 
439 	for (d = 0 ; d < bytes ; d += 256) {
440 		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
441 			     "vmovdqa64 %1,%%zmm6\n\t"
442 			     "vmovdqa64 %2,%%zmm12\n\t"
443 			     "vmovdqa64 %3,%%zmm14\n\t"
444 			     "vmovdqa64 %4,%%zmm2\n\t"
445 			     "vmovdqa64 %5,%%zmm3\n\t"
446 			     "vmovdqa64 %6,%%zmm10\n\t"
447 			     "vmovdqa64 %7,%%zmm11\n\t"
448 			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
449 			     "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
450 			     "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
451 			     "vpxorq %%zmm14,%%zmm11,%%zmm11"
452 			     :
453 			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
454 			       "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
455 			       "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
456 			       "m" (p[d+192]));
457 		/* P/Q data pages */
458 		for (z = z0-1 ; z >= start ; z--) {
459 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
460 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
461 				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
462 				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
463 				     "prefetchnta %0\n\t"
464 				     "prefetchnta %2\n\t"
465 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
466 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
467 				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
468 				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
469 				     "vpmovm2b %%k1,%%zmm5\n\t"
470 				     "vpmovm2b %%k2,%%zmm7\n\t"
471 				     "vpmovm2b %%k3,%%zmm13\n\t"
472 				     "vpmovm2b %%k4,%%zmm15\n\t"
473 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
474 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
475 				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
476 				     "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
477 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
478 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
479 				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
480 				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
481 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
482 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
483 				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
484 				     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
485 				     "vmovdqa64 %0,%%zmm5\n\t"
486 				     "vmovdqa64 %1,%%zmm7\n\t"
487 				     "vmovdqa64 %2,%%zmm13\n\t"
488 				     "vmovdqa64 %3,%%zmm15\n\t"
489 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
490 				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
491 				     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
492 				     "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
493 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
494 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
495 				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
496 				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
497 				     :
498 				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
499 				       "m" (dptr[z][d+128]),
500 				       "m" (dptr[z][d+192]));
501 		}
502 		asm volatile("prefetchnta %0\n\t"
503 			     "prefetchnta %1\n\t"
504 			     :
505 			     : "m" (q[d]), "m" (q[d+128]));
506 		/* P/Q left side optimization */
507 		for (z = start-1 ; z >= 0 ; z--) {
508 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
509 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
510 				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
511 				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
512 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
513 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
514 				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
515 				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
516 				     "vpmovm2b %%k1,%%zmm5\n\t"
517 				     "vpmovm2b %%k2,%%zmm7\n\t"
518 				     "vpmovm2b %%k3,%%zmm13\n\t"
519 				     "vpmovm2b %%k4,%%zmm15\n\t"
520 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
521 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
522 				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
523 				     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
524 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
525 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
526 				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
527 				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
528 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
529 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
530 				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
531 				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
532 				     :
533 				     : );
534 		}
535 		asm volatile("vmovntdq %%zmm2,%0\n\t"
536 			     "vmovntdq %%zmm3,%1\n\t"
537 			     "vmovntdq %%zmm10,%2\n\t"
538 			     "vmovntdq %%zmm11,%3\n\t"
539 			     "vpxorq %4,%%zmm4,%%zmm4\n\t"
540 			     "vpxorq %5,%%zmm6,%%zmm6\n\t"
541 			     "vpxorq %6,%%zmm12,%%zmm12\n\t"
542 			     "vpxorq %7,%%zmm14,%%zmm14\n\t"
543 			     "vmovntdq %%zmm4,%4\n\t"
544 			     "vmovntdq %%zmm6,%5\n\t"
545 			     "vmovntdq %%zmm12,%6\n\t"
546 			     "vmovntdq %%zmm14,%7"
547 			     :
548 			     : "m" (p[d]),  "m" (p[d+64]), "m" (p[d+128]),
549 			       "m" (p[d+192]), "m" (q[d]),  "m" (q[d+64]),
550 			       "m" (q[d+128]), "m" (q[d+192]));
551 	}
552 	asm volatile("sfence" : : : "memory");
553 	kernel_fpu_end();
554 }
555 const struct raid6_calls raid6_avx512x4 = {
556 	raid6_avx5124_gen_syndrome,
557 	raid6_avx5124_xor_syndrome,
558 	raid6_have_avx512,
559 	"avx512x4",
560 	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
561 };
562 #endif
563 
564 #endif /* CONFIG_AS_AVX512 */
565