1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (C) 2016 Gvozden Nešković. All rights reserved. 23 */ 24 25 #ifdef __amd64 26 27 #include <sys/types.h> 28 #include <sys/sunddi.h> 29 #include <sys/byteorder.h> 30 #include <sys/simd.h> 31 #include <sys/spa_checksum.h> 32 #include <zfs_fletcher.h> 33 #ifndef _KERNEL 34 #include <strings.h> 35 #endif 36 37 static void 38 fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx) 39 { 40 bzero(ctx->avx512, 4 * sizeof (zfs_fletcher_avx512_t)); 41 } 42 43 static void 44 fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) 45 { 46 static const uint64_t 47 CcA[] = { 0, 0, 1, 3, 6, 10, 15, 21 }, 48 CcB[] = { 28, 36, 44, 52, 60, 68, 76, 84 }, 49 DcA[] = { 0, 0, 0, 1, 4, 10, 20, 35 }, 50 DcB[] = { 56, 84, 120, 164, 216, 276, 344, 420 }, 51 DcC[] = { 448, 512, 576, 640, 704, 768, 832, 896 }; 52 53 uint64_t A, B, C, D; 54 uint64_t i; 55 56 A = ctx->avx512[0].v[0]; 57 B = 8 * ctx->avx512[1].v[0]; 58 C = 64 * ctx->avx512[2].v[0] - CcB[0] * ctx->avx512[1].v[0]; 59 D = 512 * ctx->avx512[3].v[0] - DcC[0] * ctx->avx512[2].v[0] + 60 DcB[0] * ctx->avx512[1].v[0]; 61 62 for (i = 1; i < 8; i++) { 63 A += ctx->avx512[0].v[i]; 64 B += 8 * ctx->avx512[1].v[i] - i * ctx->avx512[0].v[i]; 65 C += 64 * ctx->avx512[2].v[i] - CcB[i] * ctx->avx512[1].v[i] + 66 CcA[i] * ctx->avx512[0].v[i]; 67 D += 512 * ctx->avx512[3].v[i] - DcC[i] * ctx->avx512[2].v[i] + 68 DcB[i] * ctx->avx512[1].v[i] - DcA[i] * ctx->avx512[0].v[i]; 69 } 70 71 ZIO_SET_CHECKSUM(zcp, A, B, C, D); 72 } 73 74 #define FLETCHER_4_AVX512_RESTORE_CTX(ctx) \ 75 { \ 76 __asm("vmovdqu64 %0, %%zmm0" :: "m" ((ctx)->avx512[0])); \ 77 __asm("vmovdqu64 %0, %%zmm1" :: "m" ((ctx)->avx512[1])); \ 78 __asm("vmovdqu64 %0, %%zmm2" :: "m" ((ctx)->avx512[2])); \ 79 __asm("vmovdqu64 %0, %%zmm3" :: "m" ((ctx)->avx512[3])); \ 80 } 81 82 #define FLETCHER_4_AVX512_SAVE_CTX(ctx) \ 83 { \ 84 __asm("vmovdqu64 %%zmm0, %0" : "=m" ((ctx)->avx512[0])); \ 85 __asm("vmovdqu64 %%zmm1, %0" : "=m" ((ctx)->avx512[1])); \ 86 __asm("vmovdqu64 %%zmm2, %0" : "=m" ((ctx)->avx512[2])); \ 87 __asm("vmovdqu64 %%zmm3, %0" : "=m" ((ctx)->avx512[3])); \ 88 } 89 90 static void 91 fletcher_4_avx512f_native(fletcher_4_ctx_t *ctx, const void *buf, size_t size) 92 { 93 const uint32_t *ip = buf; 94 const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); 95 96 FLETCHER_4_AVX512_RESTORE_CTX(ctx); 97 98 do { 99 __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip)); 100 __asm("vpaddq %zmm4, %zmm0, %zmm0"); 101 __asm("vpaddq %zmm0, %zmm1, %zmm1"); 102 __asm("vpaddq %zmm1, %zmm2, %zmm2"); 103 __asm("vpaddq %zmm2, %zmm3, %zmm3"); 104 } while ((ip += 8) < ipend); 105 106 FLETCHER_4_AVX512_SAVE_CTX(ctx); 107 } 108 109 static void 110 fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf, 111 size_t size) 112 { 113 static const uint64_t byteswap_mask = 0xFFULL; 114 const uint32_t *ip = buf; 115 const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); 116 117 FLETCHER_4_AVX512_RESTORE_CTX(ctx); 118 119 __asm("vpbroadcastq %0, %%zmm8" :: "r" (byteswap_mask)); 120 __asm("vpsllq $8, %zmm8, %zmm9"); 121 __asm("vpsllq $16, %zmm8, %zmm10"); 122 __asm("vpsllq $24, %zmm8, %zmm11"); 123 124 do { 125 __asm("vpmovzxdq %0, %%zmm5"::"m" (*ip)); 126 127 __asm("vpsrlq $24, %zmm5, %zmm6"); 128 __asm("vpandd %zmm8, %zmm6, %zmm6"); 129 __asm("vpsrlq $8, %zmm5, %zmm7"); 130 __asm("vpandd %zmm9, %zmm7, %zmm7"); 131 __asm("vpord %zmm6, %zmm7, %zmm4"); 132 __asm("vpsllq $8, %zmm5, %zmm6"); 133 __asm("vpandd %zmm10, %zmm6, %zmm6"); 134 __asm("vpord %zmm6, %zmm4, %zmm4"); 135 __asm("vpsllq $24, %zmm5, %zmm5"); 136 __asm("vpandd %zmm11, %zmm5, %zmm5"); 137 __asm("vpord %zmm5, %zmm4, %zmm4"); 138 139 __asm("vpaddq %zmm4, %zmm0, %zmm0"); 140 __asm("vpaddq %zmm0, %zmm1, %zmm1"); 141 __asm("vpaddq %zmm1, %zmm2, %zmm2"); 142 __asm("vpaddq %zmm2, %zmm3, %zmm3"); 143 } while ((ip += 8) < ipend); 144 145 FLETCHER_4_AVX512_SAVE_CTX(ctx) 146 } 147 148 static boolean_t 149 fletcher_4_avx512f_valid(void) 150 { 151 return (kfpu_allowed() && zfs_avx512f_available()); 152 } 153 154 const fletcher_4_ops_t fletcher_4_avx512f_ops = { 155 .init_native = fletcher_4_avx512f_init, 156 .fini_native = fletcher_4_avx512f_fini, 157 .compute_native = fletcher_4_avx512f_native, 158 .init_byteswap = fletcher_4_avx512f_init, 159 .fini_byteswap = fletcher_4_avx512f_fini, 160 .compute_byteswap = fletcher_4_avx512f_byteswap, 161 .valid = fletcher_4_avx512f_valid, 162 .uses_fpu_native = B_TRUE, 163 .uses_fpu_byteswap = B_TRUE, 164 .name = "avx512f" 165 }; 166 167 static void 168 fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf, 169 size_t size) 170 { 171 static const zfs_fletcher_avx512_t mask = { 172 .v = { 173 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B, 174 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B, 175 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B, 176 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B 177 } 178 }; 179 const uint32_t *ip = buf; 180 const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); 181 182 FLETCHER_4_AVX512_RESTORE_CTX(ctx); 183 184 __asm("vmovdqu64 %0, %%zmm5" :: "m" (mask)); 185 186 do { 187 __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip)); 188 189 __asm("vpshufb %zmm5, %zmm4, %zmm4"); 190 191 __asm("vpaddq %zmm4, %zmm0, %zmm0"); 192 __asm("vpaddq %zmm0, %zmm1, %zmm1"); 193 __asm("vpaddq %zmm1, %zmm2, %zmm2"); 194 __asm("vpaddq %zmm2, %zmm3, %zmm3"); 195 } while ((ip += 8) < ipend); 196 197 FLETCHER_4_AVX512_SAVE_CTX(ctx) 198 } 199 200 static boolean_t 201 fletcher_4_avx512bw_valid(void) 202 { 203 return (kfpu_allowed() && fletcher_4_avx512f_valid() && 204 zfs_avx512bw_available()); 205 } 206 207 208 const fletcher_4_ops_t fletcher_4_avx512bw_ops = { 209 .init_native = fletcher_4_avx512f_init, 210 .fini_native = fletcher_4_avx512f_fini, 211 .compute_native = fletcher_4_avx512f_native, 212 .init_byteswap = fletcher_4_avx512f_init, 213 .fini_byteswap = fletcher_4_avx512f_fini, 214 .compute_byteswap = fletcher_4_avx512bw_byteswap, 215 .valid = fletcher_4_avx512bw_valid, 216 .uses_fpu_native = B_TRUE, 217 .uses_fpu_byteswap = B_TRUE, 218 .name = "avx512bw" 219 }; 220 221 #endif /* __amd64 */ 222