1*0886dcadSAndy Fiddaman /*
2*0886dcadSAndy Fiddaman * Implement fast Fletcher4 with SSE2,SSSE3 instructions. (x86)
3*0886dcadSAndy Fiddaman *
4*0886dcadSAndy Fiddaman * Use the 128-bit SSE2/SSSE3 SIMD instructions and registers to compute
5*0886dcadSAndy Fiddaman * Fletcher4 in two incremental 64-bit parallel accumulator streams,
6*0886dcadSAndy Fiddaman * and then combine the streams to form the final four checksum words.
7*0886dcadSAndy Fiddaman * This implementation is a derivative of the AVX SIMD implementation by
8*0886dcadSAndy Fiddaman * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
9*0886dcadSAndy Fiddaman *
10*0886dcadSAndy Fiddaman * Copyright (C) 2016 Tyler J. Stachecki.
11*0886dcadSAndy Fiddaman *
12*0886dcadSAndy Fiddaman * Authors:
13*0886dcadSAndy Fiddaman * Tyler J. Stachecki <stachecki.tyler@gmail.com>
14*0886dcadSAndy Fiddaman *
15*0886dcadSAndy Fiddaman * This software is available to you under a choice of one of two
16*0886dcadSAndy Fiddaman * licenses. You may choose to be licensed under the terms of the GNU
17*0886dcadSAndy Fiddaman * General Public License (GPL) Version 2, available from the file
18*0886dcadSAndy Fiddaman * COPYING in the main directory of this source tree, or the
19*0886dcadSAndy Fiddaman * OpenIB.org BSD license below:
20*0886dcadSAndy Fiddaman *
21*0886dcadSAndy Fiddaman * Redistribution and use in source and binary forms, with or
22*0886dcadSAndy Fiddaman * without modification, are permitted provided that the following
23*0886dcadSAndy Fiddaman * conditions are met:
24*0886dcadSAndy Fiddaman *
25*0886dcadSAndy Fiddaman * - Redistributions of source code must retain the above
26*0886dcadSAndy Fiddaman * copyright notice, this list of conditions and the following
27*0886dcadSAndy Fiddaman * disclaimer.
28*0886dcadSAndy Fiddaman *
29*0886dcadSAndy Fiddaman * - Redistributions in binary form must reproduce the above
30*0886dcadSAndy Fiddaman * copyright notice, this list of conditions and the following
31*0886dcadSAndy Fiddaman * disclaimer in the documentation and/or other materials
32*0886dcadSAndy Fiddaman * provided with the distribution.
33*0886dcadSAndy Fiddaman *
34*0886dcadSAndy Fiddaman * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35*0886dcadSAndy Fiddaman * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36*0886dcadSAndy Fiddaman * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37*0886dcadSAndy Fiddaman * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38*0886dcadSAndy Fiddaman * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39*0886dcadSAndy Fiddaman * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40*0886dcadSAndy Fiddaman * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41*0886dcadSAndy Fiddaman * SOFTWARE.
42*0886dcadSAndy Fiddaman */
43*0886dcadSAndy Fiddaman
44*0886dcadSAndy Fiddaman #ifdef __amd64
45*0886dcadSAndy Fiddaman
46*0886dcadSAndy Fiddaman #include <sys/types.h>
47*0886dcadSAndy Fiddaman #include <sys/sunddi.h>
48*0886dcadSAndy Fiddaman #include <sys/byteorder.h>
49*0886dcadSAndy Fiddaman #include <sys/simd.h>
50*0886dcadSAndy Fiddaman #include <sys/spa_checksum.h>
51*0886dcadSAndy Fiddaman #include <zfs_fletcher.h>
52*0886dcadSAndy Fiddaman #ifndef _KERNEL
53*0886dcadSAndy Fiddaman #include <strings.h>
54*0886dcadSAndy Fiddaman #endif
55*0886dcadSAndy Fiddaman
56*0886dcadSAndy Fiddaman struct zfs_fletcher_sse_array {
57*0886dcadSAndy Fiddaman uint64_t v[2] __attribute__((aligned(16)));
58*0886dcadSAndy Fiddaman };
59*0886dcadSAndy Fiddaman
60*0886dcadSAndy Fiddaman static void
fletcher_4_sse2_init(fletcher_4_ctx_t * ctx)61*0886dcadSAndy Fiddaman fletcher_4_sse2_init(fletcher_4_ctx_t *ctx)
62*0886dcadSAndy Fiddaman {
63*0886dcadSAndy Fiddaman bzero(ctx->sse, 4 * sizeof (zfs_fletcher_sse_t));
64*0886dcadSAndy Fiddaman }
65*0886dcadSAndy Fiddaman
66*0886dcadSAndy Fiddaman static void
fletcher_4_sse2_fini(fletcher_4_ctx_t * ctx,zio_cksum_t * zcp)67*0886dcadSAndy Fiddaman fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
68*0886dcadSAndy Fiddaman {
69*0886dcadSAndy Fiddaman uint64_t A, B, C, D;
70*0886dcadSAndy Fiddaman
71*0886dcadSAndy Fiddaman /*
72*0886dcadSAndy Fiddaman * The mixing matrix for checksum calculation is:
73*0886dcadSAndy Fiddaman * a = a0 + a1
74*0886dcadSAndy Fiddaman * b = 2b0 + 2b1 - a1
75*0886dcadSAndy Fiddaman * c = 4c0 - b0 + 4c1 -3b1
76*0886dcadSAndy Fiddaman * d = 8d0 - 4c0 + 8d1 - 8c1 + b1;
77*0886dcadSAndy Fiddaman *
78*0886dcadSAndy Fiddaman * c and d are multiplied by 4 and 8, respectively,
79*0886dcadSAndy Fiddaman * before spilling the vectors out to memory.
80*0886dcadSAndy Fiddaman */
81*0886dcadSAndy Fiddaman A = ctx->sse[0].v[0] + ctx->sse[0].v[1];
82*0886dcadSAndy Fiddaman B = 2 * ctx->sse[1].v[0] + 2 * ctx->sse[1].v[1] - ctx->sse[0].v[1];
83*0886dcadSAndy Fiddaman C = 4 * ctx->sse[2].v[0] - ctx->sse[1].v[0] + 4 * ctx->sse[2].v[1] -
84*0886dcadSAndy Fiddaman 3 * ctx->sse[1].v[1];
85*0886dcadSAndy Fiddaman D = 8 * ctx->sse[3].v[0] - 4 * ctx->sse[2].v[0] + 8 * ctx->sse[3].v[1] -
86*0886dcadSAndy Fiddaman 8 * ctx->sse[2].v[1] + ctx->sse[1].v[1];
87*0886dcadSAndy Fiddaman
88*0886dcadSAndy Fiddaman ZIO_SET_CHECKSUM(zcp, A, B, C, D);
89*0886dcadSAndy Fiddaman }
90*0886dcadSAndy Fiddaman
91*0886dcadSAndy Fiddaman #define FLETCHER_4_SSE_RESTORE_CTX(ctx) \
92*0886dcadSAndy Fiddaman { \
93*0886dcadSAndy Fiddaman __asm("movdqu %0, %%xmm0" :: "m" ((ctx)->sse[0])); \
94*0886dcadSAndy Fiddaman __asm("movdqu %0, %%xmm1" :: "m" ((ctx)->sse[1])); \
95*0886dcadSAndy Fiddaman __asm("movdqu %0, %%xmm2" :: "m" ((ctx)->sse[2])); \
96*0886dcadSAndy Fiddaman __asm("movdqu %0, %%xmm3" :: "m" ((ctx)->sse[3])); \
97*0886dcadSAndy Fiddaman }
98*0886dcadSAndy Fiddaman
99*0886dcadSAndy Fiddaman #define FLETCHER_4_SSE_SAVE_CTX(ctx) \
100*0886dcadSAndy Fiddaman { \
101*0886dcadSAndy Fiddaman __asm("movdqu %%xmm0, %0" : "=m" ((ctx)->sse[0])); \
102*0886dcadSAndy Fiddaman __asm("movdqu %%xmm1, %0" : "=m" ((ctx)->sse[1])); \
103*0886dcadSAndy Fiddaman __asm("movdqu %%xmm2, %0" : "=m" ((ctx)->sse[2])); \
104*0886dcadSAndy Fiddaman __asm("movdqu %%xmm3, %0" : "=m" ((ctx)->sse[3])); \
105*0886dcadSAndy Fiddaman }
106*0886dcadSAndy Fiddaman
107*0886dcadSAndy Fiddaman static void
fletcher_4_sse2_native(fletcher_4_ctx_t * ctx,const void * buf,size_t size)108*0886dcadSAndy Fiddaman fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, size_t size)
109*0886dcadSAndy Fiddaman {
110*0886dcadSAndy Fiddaman const uint64_t *ip = buf;
111*0886dcadSAndy Fiddaman const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
112*0886dcadSAndy Fiddaman
113*0886dcadSAndy Fiddaman FLETCHER_4_SSE_RESTORE_CTX(ctx);
114*0886dcadSAndy Fiddaman
115*0886dcadSAndy Fiddaman __asm("pxor %xmm4, %xmm4");
116*0886dcadSAndy Fiddaman
117*0886dcadSAndy Fiddaman do {
118*0886dcadSAndy Fiddaman __asm("movdqu %0, %%xmm5" :: "m"(*ip));
119*0886dcadSAndy Fiddaman __asm("movdqa %xmm5, %xmm6");
120*0886dcadSAndy Fiddaman __asm("punpckldq %xmm4, %xmm5");
121*0886dcadSAndy Fiddaman __asm("punpckhdq %xmm4, %xmm6");
122*0886dcadSAndy Fiddaman __asm("paddq %xmm5, %xmm0");
123*0886dcadSAndy Fiddaman __asm("paddq %xmm0, %xmm1");
124*0886dcadSAndy Fiddaman __asm("paddq %xmm1, %xmm2");
125*0886dcadSAndy Fiddaman __asm("paddq %xmm2, %xmm3");
126*0886dcadSAndy Fiddaman __asm("paddq %xmm6, %xmm0");
127*0886dcadSAndy Fiddaman __asm("paddq %xmm0, %xmm1");
128*0886dcadSAndy Fiddaman __asm("paddq %xmm1, %xmm2");
129*0886dcadSAndy Fiddaman __asm("paddq %xmm2, %xmm3");
130*0886dcadSAndy Fiddaman } while ((ip += 2) < ipend);
131*0886dcadSAndy Fiddaman
132*0886dcadSAndy Fiddaman FLETCHER_4_SSE_SAVE_CTX(ctx);
133*0886dcadSAndy Fiddaman }
134*0886dcadSAndy Fiddaman
135*0886dcadSAndy Fiddaman static void
fletcher_4_sse2_byteswap(fletcher_4_ctx_t * ctx,const void * buf,size_t size)136*0886dcadSAndy Fiddaman fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, size_t size)
137*0886dcadSAndy Fiddaman {
138*0886dcadSAndy Fiddaman const uint32_t *ip = buf;
139*0886dcadSAndy Fiddaman const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);
140*0886dcadSAndy Fiddaman
141*0886dcadSAndy Fiddaman FLETCHER_4_SSE_RESTORE_CTX(ctx);
142*0886dcadSAndy Fiddaman
143*0886dcadSAndy Fiddaman do {
144*0886dcadSAndy Fiddaman uint32_t scratch1 = BSWAP_32(ip[0]);
145*0886dcadSAndy Fiddaman uint32_t scratch2 = BSWAP_32(ip[1]);
146*0886dcadSAndy Fiddaman
147*0886dcadSAndy Fiddaman __asm("movd %0, %%xmm5" :: "r"(scratch1));
148*0886dcadSAndy Fiddaman __asm("movd %0, %%xmm6" :: "r"(scratch2));
149*0886dcadSAndy Fiddaman __asm("punpcklqdq %xmm6, %xmm5");
150*0886dcadSAndy Fiddaman __asm("paddq %xmm5, %xmm0");
151*0886dcadSAndy Fiddaman __asm("paddq %xmm0, %xmm1");
152*0886dcadSAndy Fiddaman __asm("paddq %xmm1, %xmm2");
153*0886dcadSAndy Fiddaman __asm("paddq %xmm2, %xmm3");
154*0886dcadSAndy Fiddaman } while ((ip += 2) < ipend);
155*0886dcadSAndy Fiddaman
156*0886dcadSAndy Fiddaman FLETCHER_4_SSE_SAVE_CTX(ctx);
157*0886dcadSAndy Fiddaman }
158*0886dcadSAndy Fiddaman
159*0886dcadSAndy Fiddaman static boolean_t
fletcher_4_sse2_valid(void)160*0886dcadSAndy Fiddaman fletcher_4_sse2_valid(void)
161*0886dcadSAndy Fiddaman {
162*0886dcadSAndy Fiddaman return (kfpu_allowed() && zfs_sse2_available());
163*0886dcadSAndy Fiddaman }
164*0886dcadSAndy Fiddaman
165*0886dcadSAndy Fiddaman const fletcher_4_ops_t fletcher_4_sse2_ops = {
166*0886dcadSAndy Fiddaman .init_native = fletcher_4_sse2_init,
167*0886dcadSAndy Fiddaman .fini_native = fletcher_4_sse2_fini,
168*0886dcadSAndy Fiddaman .compute_native = fletcher_4_sse2_native,
169*0886dcadSAndy Fiddaman .init_byteswap = fletcher_4_sse2_init,
170*0886dcadSAndy Fiddaman .fini_byteswap = fletcher_4_sse2_fini,
171*0886dcadSAndy Fiddaman .compute_byteswap = fletcher_4_sse2_byteswap,
172*0886dcadSAndy Fiddaman .valid = fletcher_4_sse2_valid,
173*0886dcadSAndy Fiddaman .uses_fpu_native = B_TRUE,
174*0886dcadSAndy Fiddaman .uses_fpu_byteswap = B_TRUE,
175*0886dcadSAndy Fiddaman .name = "sse2"
176*0886dcadSAndy Fiddaman };
177*0886dcadSAndy Fiddaman
178*0886dcadSAndy Fiddaman static void
fletcher_4_ssse3_byteswap(fletcher_4_ctx_t * ctx,const void * buf,size_t size)179*0886dcadSAndy Fiddaman fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, size_t size)
180*0886dcadSAndy Fiddaman {
181*0886dcadSAndy Fiddaman static const zfs_fletcher_sse_t mask = {
182*0886dcadSAndy Fiddaman .v = { 0x0405060700010203, 0x0C0D0E0F08090A0B }
183*0886dcadSAndy Fiddaman };
184*0886dcadSAndy Fiddaman
185*0886dcadSAndy Fiddaman const uint64_t *ip = buf;
186*0886dcadSAndy Fiddaman const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
187*0886dcadSAndy Fiddaman
188*0886dcadSAndy Fiddaman FLETCHER_4_SSE_RESTORE_CTX(ctx);
189*0886dcadSAndy Fiddaman
190*0886dcadSAndy Fiddaman __asm("movdqa %0, %%xmm7"::"m" (mask));
191*0886dcadSAndy Fiddaman __asm("pxor %xmm4, %xmm4");
192*0886dcadSAndy Fiddaman
193*0886dcadSAndy Fiddaman do {
194*0886dcadSAndy Fiddaman __asm("movdqu %0, %%xmm5"::"m" (*ip));
195*0886dcadSAndy Fiddaman __asm("pshufb %xmm7, %xmm5");
196*0886dcadSAndy Fiddaman __asm("movdqa %xmm5, %xmm6");
197*0886dcadSAndy Fiddaman __asm("punpckldq %xmm4, %xmm5");
198*0886dcadSAndy Fiddaman __asm("punpckhdq %xmm4, %xmm6");
199*0886dcadSAndy Fiddaman __asm("paddq %xmm5, %xmm0");
200*0886dcadSAndy Fiddaman __asm("paddq %xmm0, %xmm1");
201*0886dcadSAndy Fiddaman __asm("paddq %xmm1, %xmm2");
202*0886dcadSAndy Fiddaman __asm("paddq %xmm2, %xmm3");
203*0886dcadSAndy Fiddaman __asm("paddq %xmm6, %xmm0");
204*0886dcadSAndy Fiddaman __asm("paddq %xmm0, %xmm1");
205*0886dcadSAndy Fiddaman __asm("paddq %xmm1, %xmm2");
206*0886dcadSAndy Fiddaman __asm("paddq %xmm2, %xmm3");
207*0886dcadSAndy Fiddaman } while ((ip += 2) < ipend);
208*0886dcadSAndy Fiddaman
209*0886dcadSAndy Fiddaman FLETCHER_4_SSE_SAVE_CTX(ctx);
210*0886dcadSAndy Fiddaman }
211*0886dcadSAndy Fiddaman
fletcher_4_ssse3_valid(void)212*0886dcadSAndy Fiddaman static boolean_t fletcher_4_ssse3_valid(void)
213*0886dcadSAndy Fiddaman {
214*0886dcadSAndy Fiddaman return (zfs_sse2_available() && zfs_ssse3_available());
215*0886dcadSAndy Fiddaman }
216*0886dcadSAndy Fiddaman
217*0886dcadSAndy Fiddaman const fletcher_4_ops_t fletcher_4_ssse3_ops = {
218*0886dcadSAndy Fiddaman .init_native = fletcher_4_sse2_init,
219*0886dcadSAndy Fiddaman .fini_native = fletcher_4_sse2_fini,
220*0886dcadSAndy Fiddaman .compute_native = fletcher_4_sse2_native,
221*0886dcadSAndy Fiddaman .init_byteswap = fletcher_4_sse2_init,
222*0886dcadSAndy Fiddaman .fini_byteswap = fletcher_4_sse2_fini,
223*0886dcadSAndy Fiddaman .compute_byteswap = fletcher_4_ssse3_byteswap,
224*0886dcadSAndy Fiddaman .valid = fletcher_4_ssse3_valid,
225*0886dcadSAndy Fiddaman .uses_fpu_native = B_TRUE,
226*0886dcadSAndy Fiddaman .uses_fpu_byteswap = B_TRUE,
227*0886dcadSAndy Fiddaman .name = "ssse3"
228*0886dcadSAndy Fiddaman };
229*0886dcadSAndy Fiddaman
230*0886dcadSAndy Fiddaman #endif /* __amd64 */
231