1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
14 */
15
16 #include <sys/isa_defs.h>
17
18 #if defined(__x86_64) && HAVE_SIMD(AVX2)
19
20 #include <sys/types.h>
21 #include <sys/simd.h>
22
23 #ifdef __linux__
24 #define __asm __asm__ __volatile__
25 #endif
26
27 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
28 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
29
30 #define VR0_(REG, ...) "ymm"#REG
31 #define VR1_(_1, REG, ...) "ymm"#REG
32 #define VR2_(_1, _2, REG, ...) "ymm"#REG
33 #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG
34 #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
35 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
36 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
37 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
38
39 #define VR0(r...) VR0_(r)
40 #define VR1(r...) VR1_(r)
41 #define VR2(r...) VR2_(r, 1)
42 #define VR3(r...) VR3_(r, 1, 2)
43 #define VR4(r...) VR4_(r, 1, 2)
44 #define VR5(r...) VR5_(r, 1, 2, 3)
45 #define VR6(r...) VR6_(r, 1, 2, 3, 4)
46 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
47
48 #define R_01(REG1, REG2, ...) REG1, REG2
49 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
50 #define R_23(REG...) _R_23(REG, 1, 2, 3)
51
52 #define ZFS_ASM_BUG() ASSERT(0)
53
54 extern const uint8_t gf_clmul_mod_lt[4*256][16];
55
56 #define ELEM_SIZE 32
57
58 typedef struct v {
59 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
60 } v_t;
61
62
63 #define XOR_ACC(src, r...) \
64 { \
65 switch (REG_CNT(r)) { \
66 case 4: \
67 __asm( \
68 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
69 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
70 "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \
71 "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \
72 : : [SRC] "r" (src)); \
73 break; \
74 case 2: \
75 __asm( \
76 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
77 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
78 : : [SRC] "r" (src)); \
79 break; \
80 default: \
81 ZFS_ASM_BUG(); \
82 } \
83 }
84
85 #define XOR(r...) \
86 { \
87 switch (REG_CNT(r)) { \
88 case 8: \
89 __asm( \
90 "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \
91 "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \
92 "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \
93 "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \
94 break; \
95 case 4: \
96 __asm( \
97 "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \
98 "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \
99 break; \
100 default: \
101 ZFS_ASM_BUG(); \
102 } \
103 }
104
105 #define ZERO(r...) XOR(r, r)
106
107 #define COPY(r...) \
108 { \
109 switch (REG_CNT(r)) { \
110 case 8: \
111 __asm( \
112 "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \
113 "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \
114 "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \
115 "vmovdqa %" VR3(r) ", %" VR7(r)); \
116 break; \
117 case 4: \
118 __asm( \
119 "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \
120 "vmovdqa %" VR1(r) ", %" VR3(r)); \
121 break; \
122 default: \
123 ZFS_ASM_BUG(); \
124 } \
125 }
126
127 #define LOAD(src, r...) \
128 { \
129 switch (REG_CNT(r)) { \
130 case 4: \
131 __asm( \
132 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
133 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
134 "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \
135 "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \
136 : : [SRC] "r" (src)); \
137 break; \
138 case 2: \
139 __asm( \
140 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
141 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
142 : : [SRC] "r" (src)); \
143 break; \
144 default: \
145 ZFS_ASM_BUG(); \
146 } \
147 }
148
149 #define STORE(dst, r...) \
150 { \
151 switch (REG_CNT(r)) { \
152 case 4: \
153 __asm( \
154 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
155 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
156 "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \
157 "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \
158 : : [DST] "r" (dst)); \
159 break; \
160 case 2: \
161 __asm( \
162 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
163 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
164 : : [DST] "r" (dst)); \
165 break; \
166 default: \
167 ZFS_ASM_BUG(); \
168 } \
169 }
170
171 #define FLUSH() \
172 { \
173 __asm("vzeroupper"); \
174 }
175
176 #define MUL2_SETUP() \
177 { \
178 __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \
179 __asm("vpbroadcastq %xmm14, %ymm14"); \
180 __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \
181 }
182
183 #define _MUL2(r...) \
184 { \
185 switch (REG_CNT(r)) { \
186 case 2: \
187 __asm( \
188 "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \
189 "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \
190 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \
191 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \
192 "vpand %ymm14, %ymm12, %ymm12\n" \
193 "vpand %ymm14, %ymm13, %ymm13\n" \
194 "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \
195 "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \
196 break; \
197 default: \
198 ZFS_ASM_BUG(); \
199 } \
200 }
201
202 #define MUL2(r...) \
203 { \
204 switch (REG_CNT(r)) { \
205 case 4: \
206 _MUL2(R_01(r)); \
207 _MUL2(R_23(r)); \
208 break; \
209 case 2: \
210 _MUL2(r); \
211 break; \
212 default: \
213 ZFS_ASM_BUG(); \
214 } \
215 }
216
217 #define MUL4(r...) \
218 { \
219 MUL2(r); \
220 MUL2(r); \
221 }
222
223 #define _0f "ymm15"
224 #define _as "ymm14"
225 #define _bs "ymm13"
226 #define _ltmod "ymm12"
227 #define _ltmul "ymm11"
228 #define _ta "ymm10"
229 #define _tb "ymm15"
230
231 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
232
233 #define _MULx2(c, r...) \
234 { \
235 switch (REG_CNT(r)) { \
236 case 2: \
237 __asm( \
238 "vpbroadcastb (%[mask]), %%" _0f "\n" \
239 /* upper bits */ \
240 "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \
241 "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \
242 \
243 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \
244 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \
245 "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \
246 "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \
247 "vpand %%" _0f ", %%" _as ", %%" _as "\n" \
248 "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \
249 \
250 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \
251 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \
252 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \
253 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \
254 /* lower bits */ \
255 "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \
256 "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \
257 \
258 "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \
259 "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \
260 \
261 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \
262 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \
263 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
264 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
265 \
266 "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \
267 "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \
268 "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \
269 "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \
270 : : [mask] "r" (&_mul_mask), \
271 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \
272 break; \
273 default: \
274 ZFS_ASM_BUG(); \
275 } \
276 }
277
278 #define MUL(c, r...) \
279 { \
280 switch (REG_CNT(r)) { \
281 case 4: \
282 _MULx2(c, R_01(r)); \
283 _MULx2(c, R_23(r)); \
284 break; \
285 case 2: \
286 _MULx2(c, R_01(r)); \
287 break; \
288 default: \
289 ZFS_ASM_BUG(); \
290 } \
291 }
292
293 #define raidz_math_begin() kfpu_begin()
294 #define raidz_math_end() \
295 { \
296 FLUSH(); \
297 kfpu_end(); \
298 }
299
300
301 #define SYN_STRIDE 4
302
303 #define ZERO_STRIDE 4
304 #define ZERO_DEFINE() {}
305 #define ZERO_D 0, 1, 2, 3
306
307 #define COPY_STRIDE 4
308 #define COPY_DEFINE() {}
309 #define COPY_D 0, 1, 2, 3
310
311 #define ADD_STRIDE 4
312 #define ADD_DEFINE() {}
313 #define ADD_D 0, 1, 2, 3
314
315 #define MUL_STRIDE 4
316 #define MUL_DEFINE() {}
317 #define MUL_D 0, 1, 2, 3
318
319 #define GEN_P_STRIDE 4
320 #define GEN_P_DEFINE() {}
321 #define GEN_P_P 0, 1, 2, 3
322
323 #define GEN_PQ_STRIDE 4
324 #define GEN_PQ_DEFINE() {}
325 #define GEN_PQ_D 0, 1, 2, 3
326 #define GEN_PQ_C 4, 5, 6, 7
327
328 #define GEN_PQR_STRIDE 4
329 #define GEN_PQR_DEFINE() {}
330 #define GEN_PQR_D 0, 1, 2, 3
331 #define GEN_PQR_C 4, 5, 6, 7
332
333 #define SYN_Q_DEFINE() {}
334 #define SYN_Q_D 0, 1, 2, 3
335 #define SYN_Q_X 4, 5, 6, 7
336
337 #define SYN_R_DEFINE() {}
338 #define SYN_R_D 0, 1, 2, 3
339 #define SYN_R_X 4, 5, 6, 7
340
341 #define SYN_PQ_DEFINE() {}
342 #define SYN_PQ_D 0, 1, 2, 3
343 #define SYN_PQ_X 4, 5, 6, 7
344
345 #define REC_PQ_STRIDE 2
346 #define REC_PQ_DEFINE() {}
347 #define REC_PQ_X 0, 1
348 #define REC_PQ_Y 2, 3
349 #define REC_PQ_T 4, 5
350
351 #define SYN_PR_DEFINE() {}
352 #define SYN_PR_D 0, 1, 2, 3
353 #define SYN_PR_X 4, 5, 6, 7
354
355 #define REC_PR_STRIDE 2
356 #define REC_PR_DEFINE() {}
357 #define REC_PR_X 0, 1
358 #define REC_PR_Y 2, 3
359 #define REC_PR_T 4, 5
360
361 #define SYN_QR_DEFINE() {}
362 #define SYN_QR_D 0, 1, 2, 3
363 #define SYN_QR_X 4, 5, 6, 7
364
365 #define REC_QR_STRIDE 2
366 #define REC_QR_DEFINE() {}
367 #define REC_QR_X 0, 1
368 #define REC_QR_Y 2, 3
369 #define REC_QR_T 4, 5
370
371 #define SYN_PQR_DEFINE() {}
372 #define SYN_PQR_D 0, 1, 2, 3
373 #define SYN_PQR_X 4, 5, 6, 7
374
375 #define REC_PQR_STRIDE 2
376 #define REC_PQR_DEFINE() {}
377 #define REC_PQR_X 0, 1
378 #define REC_PQR_Y 2, 3
379 #define REC_PQR_Z 4, 5
380 #define REC_PQR_XS 6, 7
381 #define REC_PQR_YS 8, 9
382
383
384 #include <sys/vdev_raidz_impl.h>
385 #include "vdev_raidz_math_impl.h"
386
387 DEFINE_GEN_METHODS(avx2);
388 DEFINE_REC_METHODS(avx2);
389
390 static boolean_t
raidz_will_avx2_work(void)391 raidz_will_avx2_work(void)
392 {
393 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
394 }
395
396 const raidz_impl_ops_t vdev_raidz_avx2_impl = {
397 .init = NULL,
398 .fini = NULL,
399 .gen = RAIDZ_GEN_METHODS(avx2),
400 .rec = RAIDZ_REC_METHODS(avx2),
401 .is_supported = &raidz_will_avx2_work,
402 .name = "avx2"
403 };
404
405 #endif /* defined(__x86_64) && HAVE_SIMD(AVX2) */
406