xref: /illumos-gate/usr/src/common/zfs/zfs_fletcher_superscalar4.c (revision 0250c53ad267726f2438e3c6556199a0bbf588a2)
1 /*
2  * Implement fast Fletcher4 using superscalar pipelines.
3  *
4  * Use regular C code to compute
5  * Fletcher4 in four incremental 64-bit parallel accumulator streams,
6  * and then combine the streams to form the final four checksum words.
7  * This implementation is a derivative of the AVX SIMD implementation by
8  * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
9  *
10  * Copyright (C) 2016 Romain Dolbeau.
11  *
12  * Authors:
13  *	Romain Dolbeau <romain.dolbeau@atos.net>
14  *
15  * This software is available to you under a choice of one of two
16  * licenses.  You may choose to be licensed under the terms of the GNU
17  * General Public License (GPL) Version 2, available from the file
18  * COPYING in the main directory of this source tree, or the
19  * OpenIB.org BSD license below:
20  *
21  *     Redistribution and use in source and binary forms, with or
22  *     without modification, are permitted provided that the following
23  *     conditions are met:
24  *
25  *	- Redistributions of source code must retain the above
26  *	  copyright notice, this list of conditions and the following
27  *	  disclaimer.
28  *
29  *	- Redistributions in binary form must reproduce the above
30  *	  copyright notice, this list of conditions and the following
31  *	  disclaimer in the documentation and/or other materials
32  *	  provided with the distribution.
33  *
34  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41  * SOFTWARE.
42  */
43 
44 #include <sys/types.h>
45 #include <sys/sunddi.h>
46 #include <sys/byteorder.h>
47 #include <sys/simd.h>
48 #include <sys/spa_checksum.h>
49 #include <zfs_fletcher.h>
50 #ifndef _KERNEL
51 #include <strings.h>
52 #endif
53 
54 static void
55 fletcher_4_superscalar4_init(fletcher_4_ctx_t *ctx)
56 {
57 	bzero(ctx->superscalar, 4 * sizeof (zfs_fletcher_superscalar_t));
58 }
59 
60 static void
61 fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
62 {
63 	uint64_t A, B, C, D;
64 
65 	A = ctx->superscalar[0].v[0] + ctx->superscalar[0].v[1] +
66 	    ctx->superscalar[0].v[2] + ctx->superscalar[0].v[3];
67 	B = 0 - ctx->superscalar[0].v[1] - 2 * ctx->superscalar[0].v[2] -
68 	    3 * ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
69 	    4 * ctx->superscalar[1].v[1] + 4 * ctx->superscalar[1].v[2] +
70 	    4 * ctx->superscalar[1].v[3];
71 
72 	C = ctx->superscalar[0].v[2] + 3 * ctx->superscalar[0].v[3] -
73 	    6 * ctx->superscalar[1].v[0] - 10 * ctx->superscalar[1].v[1] -
74 	    14 * ctx->superscalar[1].v[2] - 18 * ctx->superscalar[1].v[3] +
75 	    16 * ctx->superscalar[2].v[0] + 16 * ctx->superscalar[2].v[1] +
76 	    16 * ctx->superscalar[2].v[2] + 16 * ctx->superscalar[2].v[3];
77 
78 	D = 0 - ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
79 	    10 * ctx->superscalar[1].v[1] + 20 * ctx->superscalar[1].v[2] +
80 	    34 * ctx->superscalar[1].v[3] - 48 * ctx->superscalar[2].v[0] -
81 	    64 * ctx->superscalar[2].v[1] - 80 * ctx->superscalar[2].v[2] -
82 	    96 * ctx->superscalar[2].v[3] + 64 * ctx->superscalar[3].v[0] +
83 	    64 * ctx->superscalar[3].v[1] + 64 * ctx->superscalar[3].v[2] +
84 	    64 * ctx->superscalar[3].v[3];
85 
86 	ZIO_SET_CHECKSUM(zcp, A, B, C, D);
87 }
88 
89 static void
90 fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
91     const void *buf, size_t size)
92 {
93 	const uint32_t *ip = buf;
94 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
95 	uint64_t a, b, c, d;
96 	uint64_t a2, b2, c2, d2;
97 	uint64_t a3, b3, c3, d3;
98 	uint64_t a4, b4, c4, d4;
99 
100 	a = ctx->superscalar[0].v[0];
101 	b = ctx->superscalar[1].v[0];
102 	c = ctx->superscalar[2].v[0];
103 	d = ctx->superscalar[3].v[0];
104 	a2 = ctx->superscalar[0].v[1];
105 	b2 = ctx->superscalar[1].v[1];
106 	c2 = ctx->superscalar[2].v[1];
107 	d2 = ctx->superscalar[3].v[1];
108 	a3 = ctx->superscalar[0].v[2];
109 	b3 = ctx->superscalar[1].v[2];
110 	c3 = ctx->superscalar[2].v[2];
111 	d3 = ctx->superscalar[3].v[2];
112 	a4 = ctx->superscalar[0].v[3];
113 	b4 = ctx->superscalar[1].v[3];
114 	c4 = ctx->superscalar[2].v[3];
115 	d4 = ctx->superscalar[3].v[3];
116 
117 	do {
118 		a += ip[0];
119 		a2 += ip[1];
120 		a3 += ip[2];
121 		a4 += ip[3];
122 		b += a;
123 		b2 += a2;
124 		b3 += a3;
125 		b4 += a4;
126 		c += b;
127 		c2 += b2;
128 		c3 += b3;
129 		c4 += b4;
130 		d += c;
131 		d2 += c2;
132 		d3 += c3;
133 		d4 += c4;
134 	} while ((ip += 4) < ipend);
135 
136 	ctx->superscalar[0].v[0] = a;
137 	ctx->superscalar[1].v[0] = b;
138 	ctx->superscalar[2].v[0] = c;
139 	ctx->superscalar[3].v[0] = d;
140 	ctx->superscalar[0].v[1] = a2;
141 	ctx->superscalar[1].v[1] = b2;
142 	ctx->superscalar[2].v[1] = c2;
143 	ctx->superscalar[3].v[1] = d2;
144 	ctx->superscalar[0].v[2] = a3;
145 	ctx->superscalar[1].v[2] = b3;
146 	ctx->superscalar[2].v[2] = c3;
147 	ctx->superscalar[3].v[2] = d3;
148 	ctx->superscalar[0].v[3] = a4;
149 	ctx->superscalar[1].v[3] = b4;
150 	ctx->superscalar[2].v[3] = c4;
151 	ctx->superscalar[3].v[3] = d4;
152 }
153 
154 static void
155 fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
156     const void *buf, size_t size)
157 {
158 	const uint32_t *ip = buf;
159 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
160 	uint64_t a, b, c, d;
161 	uint64_t a2, b2, c2, d2;
162 	uint64_t a3, b3, c3, d3;
163 	uint64_t a4, b4, c4, d4;
164 
165 	a = ctx->superscalar[0].v[0];
166 	b = ctx->superscalar[1].v[0];
167 	c = ctx->superscalar[2].v[0];
168 	d = ctx->superscalar[3].v[0];
169 	a2 = ctx->superscalar[0].v[1];
170 	b2 = ctx->superscalar[1].v[1];
171 	c2 = ctx->superscalar[2].v[1];
172 	d2 = ctx->superscalar[3].v[1];
173 	a3 = ctx->superscalar[0].v[2];
174 	b3 = ctx->superscalar[1].v[2];
175 	c3 = ctx->superscalar[2].v[2];
176 	d3 = ctx->superscalar[3].v[2];
177 	a4 = ctx->superscalar[0].v[3];
178 	b4 = ctx->superscalar[1].v[3];
179 	c4 = ctx->superscalar[2].v[3];
180 	d4 = ctx->superscalar[3].v[3];
181 
182 	do {
183 		a += BSWAP_32(ip[0]);
184 		a2 += BSWAP_32(ip[1]);
185 		a3 += BSWAP_32(ip[2]);
186 		a4 += BSWAP_32(ip[3]);
187 		b += a;
188 		b2 += a2;
189 		b3 += a3;
190 		b4 += a4;
191 		c += b;
192 		c2 += b2;
193 		c3 += b3;
194 		c4 += b4;
195 		d += c;
196 		d2 += c2;
197 		d3 += c3;
198 		d4 += c4;
199 	} while ((ip += 4) < ipend);
200 
201 	ctx->superscalar[0].v[0] = a;
202 	ctx->superscalar[1].v[0] = b;
203 	ctx->superscalar[2].v[0] = c;
204 	ctx->superscalar[3].v[0] = d;
205 	ctx->superscalar[0].v[1] = a2;
206 	ctx->superscalar[1].v[1] = b2;
207 	ctx->superscalar[2].v[1] = c2;
208 	ctx->superscalar[3].v[1] = d2;
209 	ctx->superscalar[0].v[2] = a3;
210 	ctx->superscalar[1].v[2] = b3;
211 	ctx->superscalar[2].v[2] = c3;
212 	ctx->superscalar[3].v[2] = d3;
213 	ctx->superscalar[0].v[3] = a4;
214 	ctx->superscalar[1].v[3] = b4;
215 	ctx->superscalar[2].v[3] = c4;
216 	ctx->superscalar[3].v[3] = d4;
217 }
218 
219 static boolean_t
220 fletcher_4_superscalar4_valid(void)
221 {
222 	return (B_TRUE);
223 }
224 
225 const fletcher_4_ops_t fletcher_4_superscalar4_ops = {
226 	.init_native = fletcher_4_superscalar4_init,
227 	.compute_native = fletcher_4_superscalar4_native,
228 	.fini_native = fletcher_4_superscalar4_fini,
229 	.init_byteswap = fletcher_4_superscalar4_init,
230 	.compute_byteswap = fletcher_4_superscalar4_byteswap,
231 	.fini_byteswap = fletcher_4_superscalar4_fini,
232 	.valid = fletcher_4_superscalar4_valid,
233 	.uses_fpu_native = B_FALSE,
234 	.uses_fpu_byteswap = B_FALSE,
235 	.name = "superscalar4"
236 };
237