1*22649d4dSMartin Matuska // SPDX-License-Identifier: CDDL-1.0
2*22649d4dSMartin Matuska /*
3*22649d4dSMartin Matuska * This file and its contents are supplied under the terms of the
4*22649d4dSMartin Matuska * Common Development and Distribution License ("CDDL"), version 1.0.
5*22649d4dSMartin Matuska * You may only use this file in accordance with the terms of version
6*22649d4dSMartin Matuska * 1.0 of the CDDL.
7*22649d4dSMartin Matuska *
8*22649d4dSMartin Matuska * A full copy of the text of the CDDL should have accompanied this
9*22649d4dSMartin Matuska * source. A copy of the CDDL is also available via the Internet at
10*22649d4dSMartin Matuska * https://opensource.org/license/CDDL-1.0.
11*22649d4dSMartin Matuska */
12*22649d4dSMartin Matuska
13*22649d4dSMartin Matuska /*
14*22649d4dSMartin Matuska * Copyright (c) 2026, Christos Longros.
15*22649d4dSMartin Matuska */
16*22649d4dSMartin Matuska
17*22649d4dSMartin Matuska #include <string.h>
18*22649d4dSMartin Matuska
19*22649d4dSMartin Matuska #include <sys/types.h>
20*22649d4dSMartin Matuska #include <sys/spa_checksum.h>
21*22649d4dSMartin Matuska #include "zfs_fletcher.h"
22*22649d4dSMartin Matuska
23*22649d4dSMartin Matuska #include "unit.h"
24*22649d4dSMartin Matuska
25*22649d4dSMartin Matuska /* ========== */
26*22649d4dSMartin Matuska
27*22649d4dSMartin Matuska /*
28*22649d4dSMartin Matuska * Fletcher checksums require the buffer size to be 4-byte aligned, and the
29*22649d4dSMartin Matuska * SIMD implementations process the data in wide strides. We use a buffer that
30*22649d4dSMartin Matuska * is a multiple of every implementation's stride so a single call never has a
31*22649d4dSMartin Matuska * scalar remainder, which keeps the cross-implementation comparison exact.
32*22649d4dSMartin Matuska */
33*22649d4dSMartin Matuska #define DATA_SIZE 8192
34*22649d4dSMartin Matuska
35*22649d4dSMartin Matuska static uint8_t databuf[DATA_SIZE] __attribute__((aligned(64)));
36*22649d4dSMartin Matuska
37*22649d4dSMartin Matuska /* Deterministic; distinct bytes per word so native and byteswap differ. */
38*22649d4dSMartin Matuska static void
fill_data(void)39*22649d4dSMartin Matuska fill_data(void)
40*22649d4dSMartin Matuska {
41*22649d4dSMartin Matuska for (size_t i = 0; i < DATA_SIZE; i++)
42*22649d4dSMartin Matuska databuf[i] = (uint8_t)i;
43*22649d4dSMartin Matuska }
44*22649d4dSMartin Matuska
45*22649d4dSMartin Matuska /* ========== */
46*22649d4dSMartin Matuska
47*22649d4dSMartin Matuska /* Known answers, verifiable by hand; small buffers take the scalar path. */
48*22649d4dSMartin Matuska static MunitResult
test_fletcher4_known(const MunitParameter params[],void * data)49*22649d4dSMartin Matuska test_fletcher4_known(const MunitParameter params[], void *data)
50*22649d4dSMartin Matuska {
51*22649d4dSMartin Matuska (void) params, (void) data;
52*22649d4dSMartin Matuska zio_cksum_t zc;
53*22649d4dSMartin Matuska
54*22649d4dSMartin Matuska /* A single word w yields {w, w, w, w}. */
55*22649d4dSMartin Matuska const uint32_t one[1] = { 0x01020304u };
56*22649d4dSMartin Matuska fletcher_4_native(one, sizeof (one), NULL, &zc);
57*22649d4dSMartin Matuska unit_eq(zc.zc_word[0], 0x01020304ULL);
58*22649d4dSMartin Matuska unit_eq(zc.zc_word[1], 0x01020304ULL);
59*22649d4dSMartin Matuska unit_eq(zc.zc_word[2], 0x01020304ULL);
60*22649d4dSMartin Matuska unit_eq(zc.zc_word[3], 0x01020304ULL);
61*22649d4dSMartin Matuska
62*22649d4dSMartin Matuska /* Words {1, 2} yield {1+2, 2*1+2, 3*1+2, 4*1+2}. */
63*22649d4dSMartin Matuska const uint32_t two[2] = { 1u, 2u };
64*22649d4dSMartin Matuska fletcher_4_native(two, sizeof (two), NULL, &zc);
65*22649d4dSMartin Matuska unit_eq(zc.zc_word[0], 3);
66*22649d4dSMartin Matuska unit_eq(zc.zc_word[1], 4);
67*22649d4dSMartin Matuska unit_eq(zc.zc_word[2], 5);
68*22649d4dSMartin Matuska unit_eq(zc.zc_word[3], 6);
69*22649d4dSMartin Matuska
70*22649d4dSMartin Matuska /* An empty buffer is the zero checksum. */
71*22649d4dSMartin Matuska fletcher_4_native(databuf, 0, NULL, &zc);
72*22649d4dSMartin Matuska uint64_t bits = zc.zc_word[0] | zc.zc_word[1] |
73*22649d4dSMartin Matuska zc.zc_word[2] | zc.zc_word[3];
74*22649d4dSMartin Matuska unit_zero(bits);
75*22649d4dSMartin Matuska
76*22649d4dSMartin Matuska return (MUNIT_OK);
77*22649d4dSMartin Matuska }
78*22649d4dSMartin Matuska
79*22649d4dSMartin Matuska /* Accumulating 4-byte-aligned chunks must match the single-call checksum. */
80*22649d4dSMartin Matuska static MunitResult
test_fletcher4_incremental(const MunitParameter params[],void * data)81*22649d4dSMartin Matuska test_fletcher4_incremental(const MunitParameter params[], void *data)
82*22649d4dSMartin Matuska {
83*22649d4dSMartin Matuska (void) params, (void) data;
84*22649d4dSMartin Matuska fill_data();
85*22649d4dSMartin Matuska
86*22649d4dSMartin Matuska zio_cksum_t once, inc;
87*22649d4dSMartin Matuska fletcher_4_native(databuf, DATA_SIZE, NULL, &once);
88*22649d4dSMartin Matuska
89*22649d4dSMartin Matuska fletcher_init(&inc);
90*22649d4dSMartin Matuska (void) fletcher_4_incremental_native(databuf, 2048, &inc);
91*22649d4dSMartin Matuska (void) fletcher_4_incremental_native(databuf + 2048, 2048, &inc);
92*22649d4dSMartin Matuska (void) fletcher_4_incremental_native(databuf + 4096, DATA_SIZE - 4096,
93*22649d4dSMartin Matuska &inc);
94*22649d4dSMartin Matuska unit_true(ZIO_CHECKSUM_EQUAL(once, inc));
95*22649d4dSMartin Matuska
96*22649d4dSMartin Matuska return (MUNIT_OK);
97*22649d4dSMartin Matuska }
98*22649d4dSMartin Matuska
99*22649d4dSMartin Matuska /* varsize equals native when aligned; drops the trailing size % 4 bytes. */
100*22649d4dSMartin Matuska static MunitResult
test_fletcher4_varsize(const MunitParameter params[],void * data)101*22649d4dSMartin Matuska test_fletcher4_varsize(const MunitParameter params[], void *data)
102*22649d4dSMartin Matuska {
103*22649d4dSMartin Matuska (void) params, (void) data;
104*22649d4dSMartin Matuska fill_data();
105*22649d4dSMartin Matuska
106*22649d4dSMartin Matuska zio_cksum_t native, var;
107*22649d4dSMartin Matuska fletcher_4_native(databuf, 4096, NULL, &native);
108*22649d4dSMartin Matuska fletcher_4_native_varsize(databuf, 4096, &var);
109*22649d4dSMartin Matuska unit_true(ZIO_CHECKSUM_EQUAL(native, var));
110*22649d4dSMartin Matuska
111*22649d4dSMartin Matuska zio_cksum_t var_unaligned, var_trunc;
112*22649d4dSMartin Matuska fletcher_4_native_varsize(databuf, 4098, &var_unaligned);
113*22649d4dSMartin Matuska fletcher_4_native_varsize(databuf, 4096, &var_trunc);
114*22649d4dSMartin Matuska unit_true(ZIO_CHECKSUM_EQUAL(var_unaligned, var_trunc));
115*22649d4dSMartin Matuska
116*22649d4dSMartin Matuska return (MUNIT_OK);
117*22649d4dSMartin Matuska }
118*22649d4dSMartin Matuska
119*22649d4dSMartin Matuska /* Native and byteswapped checksums differ on byte-asymmetric data. */
120*22649d4dSMartin Matuska static MunitResult
test_fletcher4_byteswap(const MunitParameter params[],void * data)121*22649d4dSMartin Matuska test_fletcher4_byteswap(const MunitParameter params[], void *data)
122*22649d4dSMartin Matuska {
123*22649d4dSMartin Matuska (void) params, (void) data;
124*22649d4dSMartin Matuska fill_data();
125*22649d4dSMartin Matuska
126*22649d4dSMartin Matuska zio_cksum_t native, swap;
127*22649d4dSMartin Matuska fletcher_4_native(databuf, DATA_SIZE, NULL, &native);
128*22649d4dSMartin Matuska fletcher_4_byteswap(databuf, DATA_SIZE, NULL, &swap);
129*22649d4dSMartin Matuska unit_false(ZIO_CHECKSUM_EQUAL(native, swap));
130*22649d4dSMartin Matuska
131*22649d4dSMartin Matuska return (MUNIT_OK);
132*22649d4dSMartin Matuska }
133*22649d4dSMartin Matuska
134*22649d4dSMartin Matuska /* Every supported implementation must agree; unsupported ones are skipped. */
135*22649d4dSMartin Matuska static MunitResult
test_fletcher4_impls(const MunitParameter params[],void * data)136*22649d4dSMartin Matuska test_fletcher4_impls(const MunitParameter params[], void *data)
137*22649d4dSMartin Matuska {
138*22649d4dSMartin Matuska (void) params, (void) data;
139*22649d4dSMartin Matuska fill_data();
140*22649d4dSMartin Matuska
141*22649d4dSMartin Matuska static const char *const impls[] = {
142*22649d4dSMartin Matuska "scalar", "superscalar", "superscalar4",
143*22649d4dSMartin Matuska "sse2", "ssse3", "avx2", "avx512f", "avx512bw",
144*22649d4dSMartin Matuska "aarch64_neon", NULL,
145*22649d4dSMartin Matuska };
146*22649d4dSMartin Matuska
147*22649d4dSMartin Matuska /* The generic implementations are always available. */
148*22649d4dSMartin Matuska unit_eq(fletcher_4_impl_set("scalar"), 0);
149*22649d4dSMartin Matuska unit_eq(fletcher_4_impl_set("superscalar"), 0);
150*22649d4dSMartin Matuska unit_eq(fletcher_4_impl_set("superscalar4"), 0);
151*22649d4dSMartin Matuska
152*22649d4dSMartin Matuska zio_cksum_t ref = { { 0 } };
153*22649d4dSMartin Matuska boolean_t have_ref = B_FALSE;
154*22649d4dSMartin Matuska
155*22649d4dSMartin Matuska for (int i = 0; impls[i] != NULL; i++) {
156*22649d4dSMartin Matuska if (fletcher_4_impl_set(impls[i]) != 0)
157*22649d4dSMartin Matuska continue; /* not supported on this host */
158*22649d4dSMartin Matuska
159*22649d4dSMartin Matuska zio_cksum_t zc;
160*22649d4dSMartin Matuska fletcher_4_native(databuf, DATA_SIZE, NULL, &zc);
161*22649d4dSMartin Matuska
162*22649d4dSMartin Matuska if (!have_ref) {
163*22649d4dSMartin Matuska ref = zc;
164*22649d4dSMartin Matuska have_ref = B_TRUE;
165*22649d4dSMartin Matuska } else {
166*22649d4dSMartin Matuska unit_true(ZIO_CHECKSUM_EQUAL(ref, zc));
167*22649d4dSMartin Matuska }
168*22649d4dSMartin Matuska }
169*22649d4dSMartin Matuska
170*22649d4dSMartin Matuska (void) fletcher_4_impl_set("fastest");
171*22649d4dSMartin Matuska return (MUNIT_OK);
172*22649d4dSMartin Matuska }
173*22649d4dSMartin Matuska
174*22649d4dSMartin Matuska /* Fletcher-2 known answers, verifiable by hand. It folds 64-bit words. */
175*22649d4dSMartin Matuska static MunitResult
test_fletcher2_known(const MunitParameter params[],void * data)176*22649d4dSMartin Matuska test_fletcher2_known(const MunitParameter params[], void *data)
177*22649d4dSMartin Matuska {
178*22649d4dSMartin Matuska (void) params, (void) data;
179*22649d4dSMartin Matuska zio_cksum_t zc;
180*22649d4dSMartin Matuska
181*22649d4dSMartin Matuska /* One pair {x, y} yields {x, y, x, y}. */
182*22649d4dSMartin Matuska const uint64_t pair[2] = { 1, 2 };
183*22649d4dSMartin Matuska fletcher_2_native(pair, sizeof (pair), NULL, &zc);
184*22649d4dSMartin Matuska unit_eq(zc.zc_word[0], 1);
185*22649d4dSMartin Matuska unit_eq(zc.zc_word[1], 2);
186*22649d4dSMartin Matuska unit_eq(zc.zc_word[2], 1);
187*22649d4dSMartin Matuska unit_eq(zc.zc_word[3], 2);
188*22649d4dSMartin Matuska
189*22649d4dSMartin Matuska /* Two pairs {w, x, y, z} yield {w+y, x+z, 2w+y, 2x+z}. */
190*22649d4dSMartin Matuska const uint64_t pairs[4] = { 1, 2, 3, 4 };
191*22649d4dSMartin Matuska fletcher_2_native(pairs, sizeof (pairs), NULL, &zc);
192*22649d4dSMartin Matuska unit_eq(zc.zc_word[0], 4);
193*22649d4dSMartin Matuska unit_eq(zc.zc_word[1], 6);
194*22649d4dSMartin Matuska unit_eq(zc.zc_word[2], 5);
195*22649d4dSMartin Matuska unit_eq(zc.zc_word[3], 8);
196*22649d4dSMartin Matuska
197*22649d4dSMartin Matuska /* An empty buffer is the zero checksum. */
198*22649d4dSMartin Matuska fletcher_2_native(databuf, 0, NULL, &zc);
199*22649d4dSMartin Matuska uint64_t bits = zc.zc_word[0] | zc.zc_word[1] |
200*22649d4dSMartin Matuska zc.zc_word[2] | zc.zc_word[3];
201*22649d4dSMartin Matuska unit_zero(bits);
202*22649d4dSMartin Matuska
203*22649d4dSMartin Matuska return (MUNIT_OK);
204*22649d4dSMartin Matuska }
205*22649d4dSMartin Matuska
206*22649d4dSMartin Matuska /* Fletcher-2: the incremental path must match the single call. */
207*22649d4dSMartin Matuska static MunitResult
test_fletcher2_incremental(const MunitParameter params[],void * data)208*22649d4dSMartin Matuska test_fletcher2_incremental(const MunitParameter params[], void *data)
209*22649d4dSMartin Matuska {
210*22649d4dSMartin Matuska (void) params, (void) data;
211*22649d4dSMartin Matuska fill_data();
212*22649d4dSMartin Matuska
213*22649d4dSMartin Matuska zio_cksum_t once, inc;
214*22649d4dSMartin Matuska fletcher_2_native(databuf, DATA_SIZE, NULL, &once);
215*22649d4dSMartin Matuska
216*22649d4dSMartin Matuska fletcher_init(&inc);
217*22649d4dSMartin Matuska (void) fletcher_2_incremental_native(databuf, 4096, &inc);
218*22649d4dSMartin Matuska (void) fletcher_2_incremental_native(databuf + 4096, DATA_SIZE - 4096,
219*22649d4dSMartin Matuska &inc);
220*22649d4dSMartin Matuska unit_true(ZIO_CHECKSUM_EQUAL(once, inc));
221*22649d4dSMartin Matuska
222*22649d4dSMartin Matuska return (MUNIT_OK);
223*22649d4dSMartin Matuska }
224*22649d4dSMartin Matuska
225*22649d4dSMartin Matuska /* ========== */
226*22649d4dSMartin Matuska
227*22649d4dSMartin Matuska static const MunitTest fletcher_tests[] = {
228*22649d4dSMartin Matuska UNIT_TEST("fletcher4_known", test_fletcher4_known),
229*22649d4dSMartin Matuska UNIT_TEST("fletcher4_incremental", test_fletcher4_incremental),
230*22649d4dSMartin Matuska UNIT_TEST("fletcher4_varsize", test_fletcher4_varsize),
231*22649d4dSMartin Matuska UNIT_TEST("fletcher4_byteswap", test_fletcher4_byteswap),
232*22649d4dSMartin Matuska UNIT_TEST("fletcher4_impls", test_fletcher4_impls),
233*22649d4dSMartin Matuska UNIT_TEST("fletcher2_known", test_fletcher2_known),
234*22649d4dSMartin Matuska UNIT_TEST("fletcher2_incremental", test_fletcher2_incremental),
235*22649d4dSMartin Matuska { 0 },
236*22649d4dSMartin Matuska };
237*22649d4dSMartin Matuska
238*22649d4dSMartin Matuska static const MunitSuite fletcher_test_suite = {
239*22649d4dSMartin Matuska "fletcher.",
240*22649d4dSMartin Matuska fletcher_tests,
241*22649d4dSMartin Matuska NULL,
242*22649d4dSMartin Matuska 1,
243*22649d4dSMartin Matuska MUNIT_SUITE_OPTION_NONE,
244*22649d4dSMartin Matuska };
245*22649d4dSMartin Matuska
246*22649d4dSMartin Matuska int
main(int argc,char ** argv)247*22649d4dSMartin Matuska main(int argc, char **argv)
248*22649d4dSMartin Matuska {
249*22649d4dSMartin Matuska fletcher_4_init();
250*22649d4dSMartin Matuska int ret = munit_suite_main(&fletcher_test_suite, NULL, argc, argv);
251*22649d4dSMartin Matuska fletcher_4_fini();
252*22649d4dSMartin Matuska return (ret);
253*22649d4dSMartin Matuska }
254