1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
24 */
25
26 #include <sys/isa_defs.h>
27
28 #if defined(__x86_64) && HAVE_SIMD(AVX2)
29
30 #include <sys/types.h>
31 #include <sys/simd.h>
32
33 #ifdef __linux__
34 #define __asm __asm__ __volatile__
35 #endif
36
37 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
38 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
39
40 #define VR0_(REG, ...) "ymm"#REG
41 #define VR1_(_1, REG, ...) "ymm"#REG
42 #define VR2_(_1, _2, REG, ...) "ymm"#REG
43 #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG
44 #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
45 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
46 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
47 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
48
49 #define VR0(r...) VR0_(r)
50 #define VR1(r...) VR1_(r)
51 #define VR2(r...) VR2_(r, 1)
52 #define VR3(r...) VR3_(r, 1, 2)
53 #define VR4(r...) VR4_(r, 1, 2)
54 #define VR5(r...) VR5_(r, 1, 2, 3)
55 #define VR6(r...) VR6_(r, 1, 2, 3, 4)
56 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
57
58 #define R_01(REG1, REG2, ...) REG1, REG2
59 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
60 #define R_23(REG...) _R_23(REG, 1, 2, 3)
61
62 #define ZFS_ASM_BUG() ASSERT(0)
63
64 extern const uint8_t gf_clmul_mod_lt[4*256][16];
65
66 #define ELEM_SIZE 32
67
68 typedef struct v {
69 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
70 } v_t;
71
72
73 #define XOR_ACC(src, r...) \
74 { \
75 switch (REG_CNT(r)) { \
76 case 4: \
77 __asm( \
78 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
79 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
80 "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \
81 "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \
82 : : [SRC] "r" (src)); \
83 break; \
84 case 2: \
85 __asm( \
86 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
87 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
88 : : [SRC] "r" (src)); \
89 break; \
90 default: \
91 ZFS_ASM_BUG(); \
92 } \
93 }
94
95 #define XOR(r...) \
96 { \
97 switch (REG_CNT(r)) { \
98 case 8: \
99 __asm( \
100 "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \
101 "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \
102 "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \
103 "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \
104 break; \
105 case 4: \
106 __asm( \
107 "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \
108 "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \
109 break; \
110 default: \
111 ZFS_ASM_BUG(); \
112 } \
113 }
114
115 #define ZERO(r...) XOR(r, r)
116
117 #define COPY(r...) \
118 { \
119 switch (REG_CNT(r)) { \
120 case 8: \
121 __asm( \
122 "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \
123 "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \
124 "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \
125 "vmovdqa %" VR3(r) ", %" VR7(r)); \
126 break; \
127 case 4: \
128 __asm( \
129 "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \
130 "vmovdqa %" VR1(r) ", %" VR3(r)); \
131 break; \
132 default: \
133 ZFS_ASM_BUG(); \
134 } \
135 }
136
137 #define LOAD(src, r...) \
138 { \
139 switch (REG_CNT(r)) { \
140 case 4: \
141 __asm( \
142 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
143 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
144 "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \
145 "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \
146 : : [SRC] "r" (src)); \
147 break; \
148 case 2: \
149 __asm( \
150 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
151 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
152 : : [SRC] "r" (src)); \
153 break; \
154 default: \
155 ZFS_ASM_BUG(); \
156 } \
157 }
158
159 #define STORE(dst, r...) \
160 { \
161 switch (REG_CNT(r)) { \
162 case 4: \
163 __asm( \
164 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
165 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
166 "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \
167 "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \
168 : : [DST] "r" (dst)); \
169 break; \
170 case 2: \
171 __asm( \
172 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
173 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
174 : : [DST] "r" (dst)); \
175 break; \
176 default: \
177 ZFS_ASM_BUG(); \
178 } \
179 }
180
181 #define FLUSH() \
182 { \
183 __asm("vzeroupper"); \
184 }
185
186 #define MUL2_SETUP() \
187 { \
188 __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \
189 __asm("vpbroadcastq %xmm14, %ymm14"); \
190 __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \
191 }
192
193 #define _MUL2(r...) \
194 { \
195 switch (REG_CNT(r)) { \
196 case 2: \
197 __asm( \
198 "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \
199 "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \
200 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \
201 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \
202 "vpand %ymm14, %ymm12, %ymm12\n" \
203 "vpand %ymm14, %ymm13, %ymm13\n" \
204 "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \
205 "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \
206 break; \
207 default: \
208 ZFS_ASM_BUG(); \
209 } \
210 }
211
212 #define MUL2(r...) \
213 { \
214 switch (REG_CNT(r)) { \
215 case 4: \
216 _MUL2(R_01(r)); \
217 _MUL2(R_23(r)); \
218 break; \
219 case 2: \
220 _MUL2(r); \
221 break; \
222 default: \
223 ZFS_ASM_BUG(); \
224 } \
225 }
226
227 #define MUL4(r...) \
228 { \
229 MUL2(r); \
230 MUL2(r); \
231 }
232
233 #define _0f "ymm15"
234 #define _as "ymm14"
235 #define _bs "ymm13"
236 #define _ltmod "ymm12"
237 #define _ltmul "ymm11"
238 #define _ta "ymm10"
239 #define _tb "ymm15"
240
241 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
242
243 #define _MULx2(c, r...) \
244 { \
245 switch (REG_CNT(r)) { \
246 case 2: \
247 __asm( \
248 "vpbroadcastb (%[mask]), %%" _0f "\n" \
249 /* upper bits */ \
250 "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \
251 "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \
252 \
253 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \
254 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \
255 "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \
256 "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \
257 "vpand %%" _0f ", %%" _as ", %%" _as "\n" \
258 "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \
259 \
260 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \
261 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \
262 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \
263 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \
264 /* lower bits */ \
265 "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \
266 "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \
267 \
268 "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \
269 "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \
270 \
271 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \
272 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \
273 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
274 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
275 \
276 "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \
277 "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \
278 "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \
279 "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \
280 : : [mask] "r" (&_mul_mask), \
281 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \
282 break; \
283 default: \
284 ZFS_ASM_BUG(); \
285 } \
286 }
287
288 #define MUL(c, r...) \
289 { \
290 switch (REG_CNT(r)) { \
291 case 4: \
292 _MULx2(c, R_01(r)); \
293 _MULx2(c, R_23(r)); \
294 break; \
295 case 2: \
296 _MULx2(c, R_01(r)); \
297 break; \
298 default: \
299 ZFS_ASM_BUG(); \
300 } \
301 }
302
303 #define raidz_math_begin() kfpu_begin()
304 #define raidz_math_end() \
305 { \
306 FLUSH(); \
307 kfpu_end(); \
308 }
309
310
311 #define SYN_STRIDE 4
312
313 #define ZERO_STRIDE 4
314 #define ZERO_DEFINE() {}
315 #define ZERO_D 0, 1, 2, 3
316
317 #define COPY_STRIDE 4
318 #define COPY_DEFINE() {}
319 #define COPY_D 0, 1, 2, 3
320
321 #define ADD_STRIDE 4
322 #define ADD_DEFINE() {}
323 #define ADD_D 0, 1, 2, 3
324
325 #define MUL_STRIDE 4
326 #define MUL_DEFINE() {}
327 #define MUL_D 0, 1, 2, 3
328
329 #define GEN_P_STRIDE 4
330 #define GEN_P_DEFINE() {}
331 #define GEN_P_P 0, 1, 2, 3
332
333 #define GEN_PQ_STRIDE 4
334 #define GEN_PQ_DEFINE() {}
335 #define GEN_PQ_D 0, 1, 2, 3
336 #define GEN_PQ_C 4, 5, 6, 7
337
338 #define GEN_PQR_STRIDE 4
339 #define GEN_PQR_DEFINE() {}
340 #define GEN_PQR_D 0, 1, 2, 3
341 #define GEN_PQR_C 4, 5, 6, 7
342
343 #define SYN_Q_DEFINE() {}
344 #define SYN_Q_D 0, 1, 2, 3
345 #define SYN_Q_X 4, 5, 6, 7
346
347 #define SYN_R_DEFINE() {}
348 #define SYN_R_D 0, 1, 2, 3
349 #define SYN_R_X 4, 5, 6, 7
350
351 #define SYN_PQ_DEFINE() {}
352 #define SYN_PQ_D 0, 1, 2, 3
353 #define SYN_PQ_X 4, 5, 6, 7
354
355 #define REC_PQ_STRIDE 2
356 #define REC_PQ_DEFINE() {}
357 #define REC_PQ_X 0, 1
358 #define REC_PQ_Y 2, 3
359 #define REC_PQ_T 4, 5
360
361 #define SYN_PR_DEFINE() {}
362 #define SYN_PR_D 0, 1, 2, 3
363 #define SYN_PR_X 4, 5, 6, 7
364
365 #define REC_PR_STRIDE 2
366 #define REC_PR_DEFINE() {}
367 #define REC_PR_X 0, 1
368 #define REC_PR_Y 2, 3
369 #define REC_PR_T 4, 5
370
371 #define SYN_QR_DEFINE() {}
372 #define SYN_QR_D 0, 1, 2, 3
373 #define SYN_QR_X 4, 5, 6, 7
374
375 #define REC_QR_STRIDE 2
376 #define REC_QR_DEFINE() {}
377 #define REC_QR_X 0, 1
378 #define REC_QR_Y 2, 3
379 #define REC_QR_T 4, 5
380
381 #define SYN_PQR_DEFINE() {}
382 #define SYN_PQR_D 0, 1, 2, 3
383 #define SYN_PQR_X 4, 5, 6, 7
384
385 #define REC_PQR_STRIDE 2
386 #define REC_PQR_DEFINE() {}
387 #define REC_PQR_X 0, 1
388 #define REC_PQR_Y 2, 3
389 #define REC_PQR_Z 4, 5
390 #define REC_PQR_XS 6, 7
391 #define REC_PQR_YS 8, 9
392
393
394 #include <sys/vdev_raidz_impl.h>
395 #include "vdev_raidz_math_impl.h"
396
397 DEFINE_GEN_METHODS(avx2);
398 DEFINE_REC_METHODS(avx2);
399
400 static boolean_t
raidz_will_avx2_work(void)401 raidz_will_avx2_work(void)
402 {
403 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
404 }
405
406 const raidz_impl_ops_t vdev_raidz_avx2_impl = {
407 .init = NULL,
408 .fini = NULL,
409 .gen = RAIDZ_GEN_METHODS(avx2),
410 .rec = RAIDZ_REC_METHODS(avx2),
411 .is_supported = &raidz_will_avx2_work,
412 .name = "avx2"
413 };
414
415 #endif /* defined(__x86_64) && HAVE_SIMD(AVX2) */
416