1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (C) 2016 Gvozden Nešković. All rights reserved. 24 */ 25 26 #include <sys/isa_defs.h> 27 28 #if defined(__x86_64) && HAVE_SIMD(AVX2) 29 30 #include <sys/types.h> 31 #include <sys/simd.h> 32 33 #ifdef __linux__ 34 #define __asm __asm__ __volatile__ 35 #endif 36 37 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N 38 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) 39 40 #define VR0_(REG, ...) "ymm"#REG 41 #define VR1_(_1, REG, ...) "ymm"#REG 42 #define VR2_(_1, _2, REG, ...) "ymm"#REG 43 #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG 44 #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG 45 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG 46 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG 47 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG 48 49 #define VR0(r...) VR0_(r) 50 #define VR1(r...) VR1_(r) 51 #define VR2(r...) VR2_(r, 1) 52 #define VR3(r...) VR3_(r, 1, 2) 53 #define VR4(r...) VR4_(r, 1, 2) 54 #define VR5(r...) VR5_(r, 1, 2, 3) 55 #define VR6(r...) VR6_(r, 1, 2, 3, 4) 56 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) 57 58 #define R_01(REG1, REG2, ...) REG1, REG2 59 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 60 #define R_23(REG...) _R_23(REG, 1, 2, 3) 61 62 #define ZFS_ASM_BUG() ASSERT(0) 63 64 extern const uint8_t gf_clmul_mod_lt[4*256][16]; 65 66 #define ELEM_SIZE 32 67 68 typedef struct v { 69 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); 70 } v_t; 71 72 73 #define XOR_ACC(src, r...) \ 74 { \ 75 switch (REG_CNT(r)) { \ 76 case 4: \ 77 __asm( \ 78 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ 79 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ 80 "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ 81 "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ 82 : : [SRC] "r" (src)); \ 83 break; \ 84 case 2: \ 85 __asm( \ 86 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ 87 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ 88 : : [SRC] "r" (src)); \ 89 break; \ 90 default: \ 91 ZFS_ASM_BUG(); \ 92 } \ 93 } 94 95 #define XOR(r...) \ 96 { \ 97 switch (REG_CNT(r)) { \ 98 case 8: \ 99 __asm( \ 100 "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ 101 "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ 102 "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ 103 "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ 104 break; \ 105 case 4: \ 106 __asm( \ 107 "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ 108 "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ 109 break; \ 110 default: \ 111 ZFS_ASM_BUG(); \ 112 } \ 113 } 114 115 #define ZERO(r...) XOR(r, r) 116 117 #define COPY(r...) \ 118 { \ 119 switch (REG_CNT(r)) { \ 120 case 8: \ 121 __asm( \ 122 "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \ 123 "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \ 124 "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \ 125 "vmovdqa %" VR3(r) ", %" VR7(r)); \ 126 break; \ 127 case 4: \ 128 __asm( \ 129 "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \ 130 "vmovdqa %" VR1(r) ", %" VR3(r)); \ 131 break; \ 132 default: \ 133 ZFS_ASM_BUG(); \ 134 } \ 135 } 136 137 #define LOAD(src, r...) \ 138 { \ 139 switch (REG_CNT(r)) { \ 140 case 4: \ 141 __asm( \ 142 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ 143 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ 144 "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \ 145 "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \ 146 : : [SRC] "r" (src)); \ 147 break; \ 148 case 2: \ 149 __asm( \ 150 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ 151 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ 152 : : [SRC] "r" (src)); \ 153 break; \ 154 default: \ 155 ZFS_ASM_BUG(); \ 156 } \ 157 } 158 159 #define STORE(dst, r...) \ 160 { \ 161 switch (REG_CNT(r)) { \ 162 case 4: \ 163 __asm( \ 164 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ 165 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ 166 "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \ 167 "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \ 168 : : [DST] "r" (dst)); \ 169 break; \ 170 case 2: \ 171 __asm( \ 172 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ 173 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ 174 : : [DST] "r" (dst)); \ 175 break; \ 176 default: \ 177 ZFS_ASM_BUG(); \ 178 } \ 179 } 180 181 #define FLUSH() \ 182 { \ 183 __asm("vzeroupper"); \ 184 } 185 186 #define MUL2_SETUP() \ 187 { \ 188 __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ 189 __asm("vpbroadcastq %xmm14, %ymm14"); \ 190 __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \ 191 } 192 193 #define _MUL2(r...) \ 194 { \ 195 switch (REG_CNT(r)) { \ 196 case 2: \ 197 __asm( \ 198 "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \ 199 "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \ 200 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \ 201 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \ 202 "vpand %ymm14, %ymm12, %ymm12\n" \ 203 "vpand %ymm14, %ymm13, %ymm13\n" \ 204 "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \ 205 "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \ 206 break; \ 207 default: \ 208 ZFS_ASM_BUG(); \ 209 } \ 210 } 211 212 #define MUL2(r...) \ 213 { \ 214 switch (REG_CNT(r)) { \ 215 case 4: \ 216 _MUL2(R_01(r)); \ 217 _MUL2(R_23(r)); \ 218 break; \ 219 case 2: \ 220 _MUL2(r); \ 221 break; \ 222 default: \ 223 ZFS_ASM_BUG(); \ 224 } \ 225 } 226 227 #define MUL4(r...) \ 228 { \ 229 MUL2(r); \ 230 MUL2(r); \ 231 } 232 233 #define _0f "ymm15" 234 #define _as "ymm14" 235 #define _bs "ymm13" 236 #define _ltmod "ymm12" 237 #define _ltmul "ymm11" 238 #define _ta "ymm10" 239 #define _tb "ymm15" 240 241 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; 242 243 #define _MULx2(c, r...) \ 244 { \ 245 switch (REG_CNT(r)) { \ 246 case 2: \ 247 __asm( \ 248 "vpbroadcastb (%[mask]), %%" _0f "\n" \ 249 /* upper bits */ \ 250 "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \ 251 "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \ 252 \ 253 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ 254 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ 255 "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ 256 "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ 257 "vpand %%" _0f ", %%" _as ", %%" _as "\n" \ 258 "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \ 259 \ 260 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ 261 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ 262 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ 263 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ 264 /* lower bits */ \ 265 "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \ 266 "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \ 267 \ 268 "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \ 269 "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \ 270 \ 271 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ 272 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ 273 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ 274 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ 275 \ 276 "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ 277 "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ 278 "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ 279 "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ 280 : : [mask] "r" (&_mul_mask), \ 281 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ 282 break; \ 283 default: \ 284 ZFS_ASM_BUG(); \ 285 } \ 286 } 287 288 #define MUL(c, r...) \ 289 { \ 290 switch (REG_CNT(r)) { \ 291 case 4: \ 292 _MULx2(c, R_01(r)); \ 293 _MULx2(c, R_23(r)); \ 294 break; \ 295 case 2: \ 296 _MULx2(c, R_01(r)); \ 297 break; \ 298 default: \ 299 ZFS_ASM_BUG(); \ 300 } \ 301 } 302 303 #define raidz_math_begin() kfpu_begin() 304 #define raidz_math_end() \ 305 { \ 306 FLUSH(); \ 307 kfpu_end(); \ 308 } 309 310 311 #define SYN_STRIDE 4 312 313 #define ZERO_STRIDE 4 314 #define ZERO_DEFINE() {} 315 #define ZERO_D 0, 1, 2, 3 316 317 #define COPY_STRIDE 4 318 #define COPY_DEFINE() {} 319 #define COPY_D 0, 1, 2, 3 320 321 #define ADD_STRIDE 4 322 #define ADD_DEFINE() {} 323 #define ADD_D 0, 1, 2, 3 324 325 #define MUL_STRIDE 4 326 #define MUL_DEFINE() {} 327 #define MUL_D 0, 1, 2, 3 328 329 #define GEN_P_STRIDE 4 330 #define GEN_P_DEFINE() {} 331 #define GEN_P_P 0, 1, 2, 3 332 333 #define GEN_PQ_STRIDE 4 334 #define GEN_PQ_DEFINE() {} 335 #define GEN_PQ_D 0, 1, 2, 3 336 #define GEN_PQ_C 4, 5, 6, 7 337 338 #define GEN_PQR_STRIDE 4 339 #define GEN_PQR_DEFINE() {} 340 #define GEN_PQR_D 0, 1, 2, 3 341 #define GEN_PQR_C 4, 5, 6, 7 342 343 #define SYN_Q_DEFINE() {} 344 #define SYN_Q_D 0, 1, 2, 3 345 #define SYN_Q_X 4, 5, 6, 7 346 347 #define SYN_R_DEFINE() {} 348 #define SYN_R_D 0, 1, 2, 3 349 #define SYN_R_X 4, 5, 6, 7 350 351 #define SYN_PQ_DEFINE() {} 352 #define SYN_PQ_D 0, 1, 2, 3 353 #define SYN_PQ_X 4, 5, 6, 7 354 355 #define REC_PQ_STRIDE 2 356 #define REC_PQ_DEFINE() {} 357 #define REC_PQ_X 0, 1 358 #define REC_PQ_Y 2, 3 359 #define REC_PQ_T 4, 5 360 361 #define SYN_PR_DEFINE() {} 362 #define SYN_PR_D 0, 1, 2, 3 363 #define SYN_PR_X 4, 5, 6, 7 364 365 #define REC_PR_STRIDE 2 366 #define REC_PR_DEFINE() {} 367 #define REC_PR_X 0, 1 368 #define REC_PR_Y 2, 3 369 #define REC_PR_T 4, 5 370 371 #define SYN_QR_DEFINE() {} 372 #define SYN_QR_D 0, 1, 2, 3 373 #define SYN_QR_X 4, 5, 6, 7 374 375 #define REC_QR_STRIDE 2 376 #define REC_QR_DEFINE() {} 377 #define REC_QR_X 0, 1 378 #define REC_QR_Y 2, 3 379 #define REC_QR_T 4, 5 380 381 #define SYN_PQR_DEFINE() {} 382 #define SYN_PQR_D 0, 1, 2, 3 383 #define SYN_PQR_X 4, 5, 6, 7 384 385 #define REC_PQR_STRIDE 2 386 #define REC_PQR_DEFINE() {} 387 #define REC_PQR_X 0, 1 388 #define REC_PQR_Y 2, 3 389 #define REC_PQR_Z 4, 5 390 #define REC_PQR_XS 6, 7 391 #define REC_PQR_YS 8, 9 392 393 394 #include <sys/vdev_raidz_impl.h> 395 #include "vdev_raidz_math_impl.h" 396 397 DEFINE_GEN_METHODS(avx2); 398 DEFINE_REC_METHODS(avx2); 399 400 static boolean_t 401 raidz_will_avx2_work(void) 402 { 403 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available()); 404 } 405 406 const raidz_impl_ops_t vdev_raidz_avx2_impl = { 407 .init = NULL, 408 .fini = NULL, 409 .gen = RAIDZ_GEN_METHODS(avx2), 410 .rec = RAIDZ_REC_METHODS(avx2), 411 .is_supported = &raidz_will_avx2_work, 412 .name = "avx2" 413 }; 414 415 #endif /* defined(__x86_64) && HAVE_SIMD(AVX2) */ 416