1 /*
2 * Implement fast Fletcher4 using superscalar pipelines.
3 *
4 * Use regular C code to compute
5 * Fletcher4 in four incremental 64-bit parallel accumulator streams,
6 * and then combine the streams to form the final four checksum words.
7 * This implementation is a derivative of the AVX SIMD implementation by
8 * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
9 *
10 * Copyright (C) 2016 Romain Dolbeau.
11 *
12 * Authors:
13 * Romain Dolbeau <romain.dolbeau@atos.net>
14 *
15 * This software is available to you under a choice of one of two
16 * licenses. You may choose to be licensed under the terms of the GNU
17 * General Public License (GPL) Version 2, available from the file
18 * COPYING in the main directory of this source tree, or the
19 * OpenIB.org BSD license below:
20 *
21 * Redistribution and use in source and binary forms, with or
22 * without modification, are permitted provided that the following
23 * conditions are met:
24 *
25 * - Redistributions of source code must retain the above
26 * copyright notice, this list of conditions and the following
27 * disclaimer.
28 *
29 * - Redistributions in binary form must reproduce the above
30 * copyright notice, this list of conditions and the following
31 * disclaimer in the documentation and/or other materials
32 * provided with the distribution.
33 *
34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41 * SOFTWARE.
42 */
43
44 #include <sys/types.h>
45 #include <sys/sunddi.h>
46 #include <sys/byteorder.h>
47 #include <sys/simd.h>
48 #include <sys/spa_checksum.h>
49 #include <zfs_fletcher.h>
50 #ifndef _KERNEL
51 #include <strings.h>
52 #endif
53
54 static void
fletcher_4_superscalar4_init(fletcher_4_ctx_t * ctx)55 fletcher_4_superscalar4_init(fletcher_4_ctx_t *ctx)
56 {
57 bzero(ctx->superscalar, 4 * sizeof (zfs_fletcher_superscalar_t));
58 }
59
60 static void
fletcher_4_superscalar4_fini(fletcher_4_ctx_t * ctx,zio_cksum_t * zcp)61 fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
62 {
63 uint64_t A, B, C, D;
64
65 A = ctx->superscalar[0].v[0] + ctx->superscalar[0].v[1] +
66 ctx->superscalar[0].v[2] + ctx->superscalar[0].v[3];
67 B = 0 - ctx->superscalar[0].v[1] - 2 * ctx->superscalar[0].v[2] -
68 3 * ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
69 4 * ctx->superscalar[1].v[1] + 4 * ctx->superscalar[1].v[2] +
70 4 * ctx->superscalar[1].v[3];
71
72 C = ctx->superscalar[0].v[2] + 3 * ctx->superscalar[0].v[3] -
73 6 * ctx->superscalar[1].v[0] - 10 * ctx->superscalar[1].v[1] -
74 14 * ctx->superscalar[1].v[2] - 18 * ctx->superscalar[1].v[3] +
75 16 * ctx->superscalar[2].v[0] + 16 * ctx->superscalar[2].v[1] +
76 16 * ctx->superscalar[2].v[2] + 16 * ctx->superscalar[2].v[3];
77
78 D = 0 - ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
79 10 * ctx->superscalar[1].v[1] + 20 * ctx->superscalar[1].v[2] +
80 34 * ctx->superscalar[1].v[3] - 48 * ctx->superscalar[2].v[0] -
81 64 * ctx->superscalar[2].v[1] - 80 * ctx->superscalar[2].v[2] -
82 96 * ctx->superscalar[2].v[3] + 64 * ctx->superscalar[3].v[0] +
83 64 * ctx->superscalar[3].v[1] + 64 * ctx->superscalar[3].v[2] +
84 64 * ctx->superscalar[3].v[3];
85
86 ZIO_SET_CHECKSUM(zcp, A, B, C, D);
87 }
88
89 static void
fletcher_4_superscalar4_native(fletcher_4_ctx_t * ctx,const void * buf,size_t size)90 fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
91 const void *buf, size_t size)
92 {
93 const uint32_t *ip = buf;
94 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
95 uint64_t a, b, c, d;
96 uint64_t a2, b2, c2, d2;
97 uint64_t a3, b3, c3, d3;
98 uint64_t a4, b4, c4, d4;
99
100 a = ctx->superscalar[0].v[0];
101 b = ctx->superscalar[1].v[0];
102 c = ctx->superscalar[2].v[0];
103 d = ctx->superscalar[3].v[0];
104 a2 = ctx->superscalar[0].v[1];
105 b2 = ctx->superscalar[1].v[1];
106 c2 = ctx->superscalar[2].v[1];
107 d2 = ctx->superscalar[3].v[1];
108 a3 = ctx->superscalar[0].v[2];
109 b3 = ctx->superscalar[1].v[2];
110 c3 = ctx->superscalar[2].v[2];
111 d3 = ctx->superscalar[3].v[2];
112 a4 = ctx->superscalar[0].v[3];
113 b4 = ctx->superscalar[1].v[3];
114 c4 = ctx->superscalar[2].v[3];
115 d4 = ctx->superscalar[3].v[3];
116
117 do {
118 a += ip[0];
119 a2 += ip[1];
120 a3 += ip[2];
121 a4 += ip[3];
122 b += a;
123 b2 += a2;
124 b3 += a3;
125 b4 += a4;
126 c += b;
127 c2 += b2;
128 c3 += b3;
129 c4 += b4;
130 d += c;
131 d2 += c2;
132 d3 += c3;
133 d4 += c4;
134 } while ((ip += 4) < ipend);
135
136 ctx->superscalar[0].v[0] = a;
137 ctx->superscalar[1].v[0] = b;
138 ctx->superscalar[2].v[0] = c;
139 ctx->superscalar[3].v[0] = d;
140 ctx->superscalar[0].v[1] = a2;
141 ctx->superscalar[1].v[1] = b2;
142 ctx->superscalar[2].v[1] = c2;
143 ctx->superscalar[3].v[1] = d2;
144 ctx->superscalar[0].v[2] = a3;
145 ctx->superscalar[1].v[2] = b3;
146 ctx->superscalar[2].v[2] = c3;
147 ctx->superscalar[3].v[2] = d3;
148 ctx->superscalar[0].v[3] = a4;
149 ctx->superscalar[1].v[3] = b4;
150 ctx->superscalar[2].v[3] = c4;
151 ctx->superscalar[3].v[3] = d4;
152 }
153
154 static void
fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t * ctx,const void * buf,size_t size)155 fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
156 const void *buf, size_t size)
157 {
158 const uint32_t *ip = buf;
159 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
160 uint64_t a, b, c, d;
161 uint64_t a2, b2, c2, d2;
162 uint64_t a3, b3, c3, d3;
163 uint64_t a4, b4, c4, d4;
164
165 a = ctx->superscalar[0].v[0];
166 b = ctx->superscalar[1].v[0];
167 c = ctx->superscalar[2].v[0];
168 d = ctx->superscalar[3].v[0];
169 a2 = ctx->superscalar[0].v[1];
170 b2 = ctx->superscalar[1].v[1];
171 c2 = ctx->superscalar[2].v[1];
172 d2 = ctx->superscalar[3].v[1];
173 a3 = ctx->superscalar[0].v[2];
174 b3 = ctx->superscalar[1].v[2];
175 c3 = ctx->superscalar[2].v[2];
176 d3 = ctx->superscalar[3].v[2];
177 a4 = ctx->superscalar[0].v[3];
178 b4 = ctx->superscalar[1].v[3];
179 c4 = ctx->superscalar[2].v[3];
180 d4 = ctx->superscalar[3].v[3];
181
182 do {
183 a += BSWAP_32(ip[0]);
184 a2 += BSWAP_32(ip[1]);
185 a3 += BSWAP_32(ip[2]);
186 a4 += BSWAP_32(ip[3]);
187 b += a;
188 b2 += a2;
189 b3 += a3;
190 b4 += a4;
191 c += b;
192 c2 += b2;
193 c3 += b3;
194 c4 += b4;
195 d += c;
196 d2 += c2;
197 d3 += c3;
198 d4 += c4;
199 } while ((ip += 4) < ipend);
200
201 ctx->superscalar[0].v[0] = a;
202 ctx->superscalar[1].v[0] = b;
203 ctx->superscalar[2].v[0] = c;
204 ctx->superscalar[3].v[0] = d;
205 ctx->superscalar[0].v[1] = a2;
206 ctx->superscalar[1].v[1] = b2;
207 ctx->superscalar[2].v[1] = c2;
208 ctx->superscalar[3].v[1] = d2;
209 ctx->superscalar[0].v[2] = a3;
210 ctx->superscalar[1].v[2] = b3;
211 ctx->superscalar[2].v[2] = c3;
212 ctx->superscalar[3].v[2] = d3;
213 ctx->superscalar[0].v[3] = a4;
214 ctx->superscalar[1].v[3] = b4;
215 ctx->superscalar[2].v[3] = c4;
216 ctx->superscalar[3].v[3] = d4;
217 }
218
219 static boolean_t
fletcher_4_superscalar4_valid(void)220 fletcher_4_superscalar4_valid(void)
221 {
222 return (B_TRUE);
223 }
224
225 const fletcher_4_ops_t fletcher_4_superscalar4_ops = {
226 .init_native = fletcher_4_superscalar4_init,
227 .compute_native = fletcher_4_superscalar4_native,
228 .fini_native = fletcher_4_superscalar4_fini,
229 .init_byteswap = fletcher_4_superscalar4_init,
230 .compute_byteswap = fletcher_4_superscalar4_byteswap,
231 .fini_byteswap = fletcher_4_superscalar4_fini,
232 .valid = fletcher_4_superscalar4_valid,
233 .uses_fpu_native = B_FALSE,
234 .uses_fpu_byteswap = B_FALSE,
235 .name = "superscalar4"
236 };
237