1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (C) 2016 Gvozden Nešković. All rights reserved. 23 */ 24 #include <sys/isa_defs.h> 25 26 #if defined(__amd64) 27 28 #include <sys/types.h> 29 #include <sys/simd.h> 30 31 #define __asm __asm__ __volatile__ 32 33 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N 34 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) 35 36 #define VR0_(REG, ...) "ymm"#REG 37 #define VR1_(_1, REG, ...) "ymm"#REG 38 #define VR2_(_1, _2, REG, ...) "ymm"#REG 39 #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG 40 #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG 41 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG 42 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG 43 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG 44 45 #define VR0(r...) VR0_(r) 46 #define VR1(r...) VR1_(r) 47 #define VR2(r...) VR2_(r, 1) 48 #define VR3(r...) VR3_(r, 1, 2) 49 #define VR4(r...) VR4_(r, 1, 2) 50 #define VR5(r...) VR5_(r, 1, 2, 3) 51 #define VR6(r...) VR6_(r, 1, 2, 3, 4) 52 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) 53 54 #define R_01(REG1, REG2, ...) REG1, REG2 55 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 56 #define R_23(REG...) _R_23(REG, 1, 2, 3) 57 58 #define ZFS_ASM_BUG() ASSERT(0) 59 60 extern const uint8_t gf_clmul_mod_lt[4*256][16]; 61 62 #define ELEM_SIZE 32 63 64 typedef struct v { 65 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); 66 } v_t; 67 68 69 #define XOR_ACC(src, r...) \ 70 { \ 71 switch (REG_CNT(r)) { \ 72 case 4: \ 73 __asm( \ 74 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ 75 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ 76 "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ 77 "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ 78 : : [SRC] "r" (src)); \ 79 break; \ 80 case 2: \ 81 __asm( \ 82 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ 83 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ 84 : : [SRC] "r" (src)); \ 85 break; \ 86 default: \ 87 ZFS_ASM_BUG(); \ 88 } \ 89 } 90 91 #define XOR(r...) \ 92 { \ 93 switch (REG_CNT(r)) { \ 94 case 8: \ 95 __asm( \ 96 "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ 97 "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ 98 "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ 99 "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ 100 break; \ 101 case 4: \ 102 __asm( \ 103 "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ 104 "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ 105 break; \ 106 default: \ 107 ZFS_ASM_BUG(); \ 108 } \ 109 } 110 111 #define ZERO(r...) XOR(r, r) 112 113 #define COPY(r...) \ 114 { \ 115 switch (REG_CNT(r)) { \ 116 case 8: \ 117 __asm( \ 118 "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \ 119 "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \ 120 "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \ 121 "vmovdqa %" VR3(r) ", %" VR7(r)); \ 122 break; \ 123 case 4: \ 124 __asm( \ 125 "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \ 126 "vmovdqa %" VR1(r) ", %" VR3(r)); \ 127 break; \ 128 default: \ 129 ZFS_ASM_BUG(); \ 130 } \ 131 } 132 133 #define LOAD(src, r...) \ 134 { \ 135 switch (REG_CNT(r)) { \ 136 case 4: \ 137 __asm( \ 138 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ 139 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ 140 "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \ 141 "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \ 142 : : [SRC] "r" (src)); \ 143 break; \ 144 case 2: \ 145 __asm( \ 146 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ 147 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ 148 : : [SRC] "r" (src)); \ 149 break; \ 150 default: \ 151 ZFS_ASM_BUG(); \ 152 } \ 153 } 154 155 #define STORE(dst, r...) \ 156 { \ 157 switch (REG_CNT(r)) { \ 158 case 4: \ 159 __asm( \ 160 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ 161 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ 162 "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \ 163 "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \ 164 : : [DST] "r" (dst)); \ 165 break; \ 166 case 2: \ 167 __asm( \ 168 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ 169 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ 170 : : [DST] "r" (dst)); \ 171 break; \ 172 default: \ 173 ZFS_ASM_BUG(); \ 174 } \ 175 } 176 177 #define FLUSH() \ 178 { \ 179 __asm("vzeroupper"); \ 180 } 181 182 #define MUL2_SETUP() \ 183 { \ 184 __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ 185 __asm("vpbroadcastq %xmm14, %ymm14"); \ 186 __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \ 187 } 188 189 #define _MUL2(r...) \ 190 { \ 191 switch (REG_CNT(r)) { \ 192 case 2: \ 193 __asm( \ 194 "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \ 195 "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \ 196 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \ 197 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \ 198 "vpand %ymm14, %ymm12, %ymm12\n" \ 199 "vpand %ymm14, %ymm13, %ymm13\n" \ 200 "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \ 201 "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \ 202 break; \ 203 default: \ 204 ZFS_ASM_BUG(); \ 205 } \ 206 } 207 208 #define MUL2(r...) \ 209 { \ 210 switch (REG_CNT(r)) { \ 211 case 4: \ 212 _MUL2(R_01(r)); \ 213 _MUL2(R_23(r)); \ 214 break; \ 215 case 2: \ 216 _MUL2(r); \ 217 break; \ 218 default: \ 219 ZFS_ASM_BUG(); \ 220 } \ 221 } 222 223 #define MUL4(r...) \ 224 { \ 225 MUL2(r); \ 226 MUL2(r); \ 227 } 228 229 #define _0f "ymm15" 230 #define _as "ymm14" 231 #define _bs "ymm13" 232 #define _ltmod "ymm12" 233 #define _ltmul "ymm11" 234 #define _ta "ymm10" 235 #define _tb "ymm15" 236 237 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; 238 239 #define _MULx2(c, r...) \ 240 { \ 241 switch (REG_CNT(r)) { \ 242 case 2: \ 243 __asm( \ 244 "vpbroadcastb (%[mask]), %%" _0f "\n" \ 245 /* upper bits */ \ 246 "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \ 247 "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \ 248 \ 249 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ 250 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ 251 "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ 252 "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ 253 "vpand %%" _0f ", %%" _as ", %%" _as "\n" \ 254 "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \ 255 \ 256 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ 257 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ 258 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ 259 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ 260 /* lower bits */ \ 261 "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \ 262 "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \ 263 \ 264 "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \ 265 "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \ 266 \ 267 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ 268 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ 269 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ 270 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ 271 \ 272 "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ 273 "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ 274 "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ 275 "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ 276 : : [mask] "r" (&_mul_mask), \ 277 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ 278 break; \ 279 default: \ 280 ZFS_ASM_BUG(); \ 281 } \ 282 } 283 284 #define MUL(c, r...) \ 285 { \ 286 switch (REG_CNT(r)) { \ 287 case 4: \ 288 _MULx2(c, R_01(r)); \ 289 _MULx2(c, R_23(r)); \ 290 break; \ 291 case 2: \ 292 _MULx2(c, R_01(r)); \ 293 break; \ 294 default: \ 295 ZFS_ASM_BUG(); \ 296 } \ 297 } 298 299 #define raidz_math_begin() kfpu_begin() 300 #define raidz_math_end() \ 301 { \ 302 FLUSH(); \ 303 kfpu_end(); \ 304 } 305 306 307 #define SYN_STRIDE 4 308 309 #define ZERO_STRIDE 4 310 #define ZERO_DEFINE() {} 311 #define ZERO_D 0, 1, 2, 3 312 313 #define COPY_STRIDE 4 314 #define COPY_DEFINE() {} 315 #define COPY_D 0, 1, 2, 3 316 317 #define ADD_STRIDE 4 318 #define ADD_DEFINE() {} 319 #define ADD_D 0, 1, 2, 3 320 321 #define MUL_STRIDE 4 322 #define MUL_DEFINE() {} 323 #define MUL_D 0, 1, 2, 3 324 325 #define GEN_P_STRIDE 4 326 #define GEN_P_DEFINE() {} 327 #define GEN_P_P 0, 1, 2, 3 328 329 #define GEN_PQ_STRIDE 4 330 #define GEN_PQ_DEFINE() {} 331 #define GEN_PQ_D 0, 1, 2, 3 332 #define GEN_PQ_C 4, 5, 6, 7 333 334 #define GEN_PQR_STRIDE 4 335 #define GEN_PQR_DEFINE() {} 336 #define GEN_PQR_D 0, 1, 2, 3 337 #define GEN_PQR_C 4, 5, 6, 7 338 339 #define SYN_Q_DEFINE() {} 340 #define SYN_Q_D 0, 1, 2, 3 341 #define SYN_Q_X 4, 5, 6, 7 342 343 #define SYN_R_DEFINE() {} 344 #define SYN_R_D 0, 1, 2, 3 345 #define SYN_R_X 4, 5, 6, 7 346 347 #define SYN_PQ_DEFINE() {} 348 #define SYN_PQ_D 0, 1, 2, 3 349 #define SYN_PQ_X 4, 5, 6, 7 350 351 #define REC_PQ_STRIDE 2 352 #define REC_PQ_DEFINE() {} 353 #define REC_PQ_X 0, 1 354 #define REC_PQ_Y 2, 3 355 #define REC_PQ_T 4, 5 356 357 #define SYN_PR_DEFINE() {} 358 #define SYN_PR_D 0, 1, 2, 3 359 #define SYN_PR_X 4, 5, 6, 7 360 361 #define REC_PR_STRIDE 2 362 #define REC_PR_DEFINE() {} 363 #define REC_PR_X 0, 1 364 #define REC_PR_Y 2, 3 365 #define REC_PR_T 4, 5 366 367 #define SYN_QR_DEFINE() {} 368 #define SYN_QR_D 0, 1, 2, 3 369 #define SYN_QR_X 4, 5, 6, 7 370 371 #define REC_QR_STRIDE 2 372 #define REC_QR_DEFINE() {} 373 #define REC_QR_X 0, 1 374 #define REC_QR_Y 2, 3 375 #define REC_QR_T 4, 5 376 377 #define SYN_PQR_DEFINE() {} 378 #define SYN_PQR_D 0, 1, 2, 3 379 #define SYN_PQR_X 4, 5, 6, 7 380 381 #define REC_PQR_STRIDE 2 382 #define REC_PQR_DEFINE() {} 383 #define REC_PQR_X 0, 1 384 #define REC_PQR_Y 2, 3 385 #define REC_PQR_Z 4, 5 386 #define REC_PQR_XS 6, 7 387 #define REC_PQR_YS 8, 9 388 389 390 #include <sys/vdev_raidz_impl.h> 391 #include "vdev_raidz_math_impl.h" 392 393 DEFINE_GEN_METHODS(avx2); 394 DEFINE_REC_METHODS(avx2); 395 396 static boolean_t 397 raidz_will_avx2_work(void) 398 { 399 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available()); 400 } 401 402 const raidz_impl_ops_t vdev_raidz_avx2_impl = { 403 .init = NULL, 404 .fini = NULL, 405 .gen = RAIDZ_GEN_METHODS(avx2), 406 .rec = RAIDZ_REC_METHODS(avx2), 407 .is_supported = &raidz_will_avx2_work, 408 .name = "avx2" 409 }; 410 411 #elif defined(__i386) 412 413 /* 32-bit stub for user-level fakekernel dependencies */ 414 #include <sys/vdev_raidz_impl.h> 415 const raidz_impl_ops_t vdev_raidz_avx2_impl = { 416 .init = NULL, 417 .fini = NULL, 418 .gen = NULL, 419 .rec = NULL, 420 .is_supported = NULL, 421 .name = "avx2" 422 }; 423 424 #endif /* defined(__amd64) */ 425