xref: /freebsd/sys/contrib/openzfs/module/zcommon/zfs_fletcher_superscalar4.c (revision 1b10e191f341111fad7be32ead11484dfd09b800)
1 /*
2  * Implement fast Fletcher4 using superscalar pipelines.
3  *
4  * Use regular C code to compute
5  * Fletcher4 in four incremental 64-bit parallel accumulator streams,
6  * and then combine the streams to form the final four checksum words.
7  * This implementation is a derivative of the AVX SIMD implementation by
8  * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
9  *
10  * Copyright (C) 2016 Romain Dolbeau.
11  *
12  * Authors:
13  *	Romain Dolbeau <romain.dolbeau@atos.net>
14  *
15  * This software is available to you under a choice of one of two
16  * licenses.  You may choose to be licensed under the terms of the GNU
17  * General Public License (GPL) Version 2, available from the file
18  * COPYING in the main directory of this source tree, or the
19  * OpenIB.org BSD license below:
20  *
21  *     Redistribution and use in source and binary forms, with or
22  *     without modification, are permitted provided that the following
23  *     conditions are met:
24  *
25  *	- Redistributions of source code must retain the above
26  *	  copyright notice, this list of conditions and the following
27  *	  disclaimer.
28  *
29  *	- Redistributions in binary form must reproduce the above
30  *	  copyright notice, this list of conditions and the following
31  *	  disclaimer in the documentation and/or other materials
32  *	  provided with the distribution.
33  *
34  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41  * SOFTWARE.
42  */
43 
44 #include <sys/param.h>
45 #include <sys/byteorder.h>
46 #include <sys/spa_checksum.h>
47 #include <sys/string.h>
48 #include <zfs_fletcher.h>
49 
50 ZFS_NO_SANITIZE_UNDEFINED
51 static void
52 fletcher_4_superscalar4_init(fletcher_4_ctx_t *ctx)
53 {
54 	memset(ctx->superscalar, 0, 4 * sizeof (zfs_fletcher_superscalar_t));
55 }
56 
57 ZFS_NO_SANITIZE_UNDEFINED
58 static void
59 fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
60 {
61 	uint64_t A, B, C, D;
62 
63 	A = ctx->superscalar[0].v[0] + ctx->superscalar[0].v[1] +
64 	    ctx->superscalar[0].v[2] + ctx->superscalar[0].v[3];
65 	B = 0 - ctx->superscalar[0].v[1] - 2 * ctx->superscalar[0].v[2] -
66 	    3 * ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
67 	    4 * ctx->superscalar[1].v[1] + 4 * ctx->superscalar[1].v[2] +
68 	    4 * ctx->superscalar[1].v[3];
69 
70 	C = ctx->superscalar[0].v[2] + 3 * ctx->superscalar[0].v[3] -
71 	    6 * ctx->superscalar[1].v[0] - 10 * ctx->superscalar[1].v[1] -
72 	    14 * ctx->superscalar[1].v[2] - 18 * ctx->superscalar[1].v[3] +
73 	    16 * ctx->superscalar[2].v[0] + 16 * ctx->superscalar[2].v[1] +
74 	    16 * ctx->superscalar[2].v[2] + 16 * ctx->superscalar[2].v[3];
75 
76 	D = 0 - ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
77 	    10 * ctx->superscalar[1].v[1] + 20 * ctx->superscalar[1].v[2] +
78 	    34 * ctx->superscalar[1].v[3] - 48 * ctx->superscalar[2].v[0] -
79 	    64 * ctx->superscalar[2].v[1] - 80 * ctx->superscalar[2].v[2] -
80 	    96 * ctx->superscalar[2].v[3] + 64 * ctx->superscalar[3].v[0] +
81 	    64 * ctx->superscalar[3].v[1] + 64 * ctx->superscalar[3].v[2] +
82 	    64 * ctx->superscalar[3].v[3];
83 
84 	ZIO_SET_CHECKSUM(zcp, A, B, C, D);
85 }
86 
87 ZFS_NO_SANITIZE_UNDEFINED
88 static void
89 fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
90     const void *buf, uint64_t size)
91 {
92 	const uint32_t *ip = buf;
93 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
94 	uint64_t a, b, c, d;
95 	uint64_t a2, b2, c2, d2;
96 	uint64_t a3, b3, c3, d3;
97 	uint64_t a4, b4, c4, d4;
98 
99 	a = ctx->superscalar[0].v[0];
100 	b = ctx->superscalar[1].v[0];
101 	c = ctx->superscalar[2].v[0];
102 	d = ctx->superscalar[3].v[0];
103 	a2 = ctx->superscalar[0].v[1];
104 	b2 = ctx->superscalar[1].v[1];
105 	c2 = ctx->superscalar[2].v[1];
106 	d2 = ctx->superscalar[3].v[1];
107 	a3 = ctx->superscalar[0].v[2];
108 	b3 = ctx->superscalar[1].v[2];
109 	c3 = ctx->superscalar[2].v[2];
110 	d3 = ctx->superscalar[3].v[2];
111 	a4 = ctx->superscalar[0].v[3];
112 	b4 = ctx->superscalar[1].v[3];
113 	c4 = ctx->superscalar[2].v[3];
114 	d4 = ctx->superscalar[3].v[3];
115 
116 	do {
117 		a += ip[0];
118 		a2 += ip[1];
119 		a3 += ip[2];
120 		a4 += ip[3];
121 		b += a;
122 		b2 += a2;
123 		b3 += a3;
124 		b4 += a4;
125 		c += b;
126 		c2 += b2;
127 		c3 += b3;
128 		c4 += b4;
129 		d += c;
130 		d2 += c2;
131 		d3 += c3;
132 		d4 += c4;
133 	} while ((ip += 4) < ipend);
134 
135 	ctx->superscalar[0].v[0] = a;
136 	ctx->superscalar[1].v[0] = b;
137 	ctx->superscalar[2].v[0] = c;
138 	ctx->superscalar[3].v[0] = d;
139 	ctx->superscalar[0].v[1] = a2;
140 	ctx->superscalar[1].v[1] = b2;
141 	ctx->superscalar[2].v[1] = c2;
142 	ctx->superscalar[3].v[1] = d2;
143 	ctx->superscalar[0].v[2] = a3;
144 	ctx->superscalar[1].v[2] = b3;
145 	ctx->superscalar[2].v[2] = c3;
146 	ctx->superscalar[3].v[2] = d3;
147 	ctx->superscalar[0].v[3] = a4;
148 	ctx->superscalar[1].v[3] = b4;
149 	ctx->superscalar[2].v[3] = c4;
150 	ctx->superscalar[3].v[3] = d4;
151 }
152 
153 ZFS_NO_SANITIZE_UNDEFINED
154 static void
155 fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
156     const void *buf, uint64_t size)
157 {
158 	const uint32_t *ip = buf;
159 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
160 	uint64_t a, b, c, d;
161 	uint64_t a2, b2, c2, d2;
162 	uint64_t a3, b3, c3, d3;
163 	uint64_t a4, b4, c4, d4;
164 
165 	a = ctx->superscalar[0].v[0];
166 	b = ctx->superscalar[1].v[0];
167 	c = ctx->superscalar[2].v[0];
168 	d = ctx->superscalar[3].v[0];
169 	a2 = ctx->superscalar[0].v[1];
170 	b2 = ctx->superscalar[1].v[1];
171 	c2 = ctx->superscalar[2].v[1];
172 	d2 = ctx->superscalar[3].v[1];
173 	a3 = ctx->superscalar[0].v[2];
174 	b3 = ctx->superscalar[1].v[2];
175 	c3 = ctx->superscalar[2].v[2];
176 	d3 = ctx->superscalar[3].v[2];
177 	a4 = ctx->superscalar[0].v[3];
178 	b4 = ctx->superscalar[1].v[3];
179 	c4 = ctx->superscalar[2].v[3];
180 	d4 = ctx->superscalar[3].v[3];
181 
182 	do {
183 		a += BSWAP_32(ip[0]);
184 		a2 += BSWAP_32(ip[1]);
185 		a3 += BSWAP_32(ip[2]);
186 		a4 += BSWAP_32(ip[3]);
187 		b += a;
188 		b2 += a2;
189 		b3 += a3;
190 		b4 += a4;
191 		c += b;
192 		c2 += b2;
193 		c3 += b3;
194 		c4 += b4;
195 		d += c;
196 		d2 += c2;
197 		d3 += c3;
198 		d4 += c4;
199 	} while ((ip += 4) < ipend);
200 
201 	ctx->superscalar[0].v[0] = a;
202 	ctx->superscalar[1].v[0] = b;
203 	ctx->superscalar[2].v[0] = c;
204 	ctx->superscalar[3].v[0] = d;
205 	ctx->superscalar[0].v[1] = a2;
206 	ctx->superscalar[1].v[1] = b2;
207 	ctx->superscalar[2].v[1] = c2;
208 	ctx->superscalar[3].v[1] = d2;
209 	ctx->superscalar[0].v[2] = a3;
210 	ctx->superscalar[1].v[2] = b3;
211 	ctx->superscalar[2].v[2] = c3;
212 	ctx->superscalar[3].v[2] = d3;
213 	ctx->superscalar[0].v[3] = a4;
214 	ctx->superscalar[1].v[3] = b4;
215 	ctx->superscalar[2].v[3] = c4;
216 	ctx->superscalar[3].v[3] = d4;
217 }
218 
219 static boolean_t fletcher_4_superscalar4_valid(void)
220 {
221 	return (B_TRUE);
222 }
223 
224 const fletcher_4_ops_t fletcher_4_superscalar4_ops = {
225 	.init_native = fletcher_4_superscalar4_init,
226 	.compute_native = fletcher_4_superscalar4_native,
227 	.fini_native = fletcher_4_superscalar4_fini,
228 	.init_byteswap = fletcher_4_superscalar4_init,
229 	.compute_byteswap = fletcher_4_superscalar4_byteswap,
230 	.fini_byteswap = fletcher_4_superscalar4_fini,
231 	.valid = fletcher_4_superscalar4_valid,
232 	.name = "superscalar4"
233 };
234