1*d0b3ecdcSMartin Matuska // SPDX-License-Identifier: CDDL-1.0
2*d0b3ecdcSMartin Matuska /*
3*d0b3ecdcSMartin Matuska * CDDL HEADER START
4*d0b3ecdcSMartin Matuska *
5*d0b3ecdcSMartin Matuska * This file and its contents are supplied under the terms of the Common
6*d0b3ecdcSMartin Matuska * Development and Distribution License ("CDDL"), version 1.0. You may only use
7*d0b3ecdcSMartin Matuska * this file in accordance with the terms of version 1.0 of the CDDL.
8*d0b3ecdcSMartin Matuska *
9*d0b3ecdcSMartin Matuska * A full copy of the text of the CDDL should have accompanied this source. A
10*d0b3ecdcSMartin Matuska * copy of the CDDL is also available via the Internet at
11*d0b3ecdcSMartin Matuska * http://www.illumos.org/license/CDDL.
12*d0b3ecdcSMartin Matuska *
13*d0b3ecdcSMartin Matuska * CDDL HEADER END
14*d0b3ecdcSMartin Matuska */
15*d0b3ecdcSMartin Matuska
16*d0b3ecdcSMartin Matuska /*
17*d0b3ecdcSMartin Matuska * Copyright (c) 2026 by Garth Snyder. All rights reserved.
18*d0b3ecdcSMartin Matuska */
19*d0b3ecdcSMartin Matuska
20*d0b3ecdcSMartin Matuska #include <assert.h>
21*d0b3ecdcSMartin Matuska #include <stddef.h>
22*d0b3ecdcSMartin Matuska #include <stdint.h>
23*d0b3ecdcSMartin Matuska #include <sys/byteorder.h>
24*d0b3ecdcSMartin Matuska #include <sys/spa_checksum.h>
25*d0b3ecdcSMartin Matuska #include <sys/stdtypes.h>
26*d0b3ecdcSMartin Matuska #include <sys/types.h>
27*d0b3ecdcSMartin Matuska #include <sys/zfs_ioctl.h>
28*d0b3ecdcSMartin Matuska #include <zfs_fletcher.h>
29*d0b3ecdcSMartin Matuska
30*d0b3ecdcSMartin Matuska #include "zstream_modules.h"
31*d0b3ecdcSMartin Matuska #include "zstream_util.h"
32*d0b3ecdcSMartin Matuska
33*d0b3ecdcSMartin Matuska #define CK_OFFSET offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum)
34*d0b3ecdcSMartin Matuska #define END_CK_OFFSET offsetof(dmu_replay_record_t, drr_u.drr_end.drr_checksum)
35*d0b3ecdcSMartin Matuska
36*d0b3ecdcSMartin Matuska typedef enum { F4_SET, F4_VALIDATE } fletcher4_op_t;
37*d0b3ecdcSMartin Matuska
38*d0b3ecdcSMartin Matuska typedef zio_cksum_t fletcher4_context_t;
39*d0b3ecdcSMartin Matuska
40*d0b3ecdcSMartin Matuska static fletcher4_context_t fletcher4_contexts[MAX_FLETCHER_4];
41*d0b3ecdcSMartin Matuska static int next_context = 0;
42*d0b3ecdcSMartin Matuska
43*d0b3ecdcSMartin Matuska static inline int
fletcher_4_incremental(boolean_t swap,void * buff,size_t size,void * cksum)44*d0b3ecdcSMartin Matuska fletcher_4_incremental(boolean_t swap, void *buff, size_t size, void *cksum)
45*d0b3ecdcSMartin Matuska {
46*d0b3ecdcSMartin Matuska if (swap) {
47*d0b3ecdcSMartin Matuska return (fletcher_4_incremental_byteswap(buff, size, cksum));
48*d0b3ecdcSMartin Matuska } else {
49*d0b3ecdcSMartin Matuska return (fletcher_4_incremental_native(buff, size, cksum));
50*d0b3ecdcSMartin Matuska }
51*d0b3ecdcSMartin Matuska }
52*d0b3ecdcSMartin Matuska
53*d0b3ecdcSMartin Matuska /*
54*d0b3ecdcSMartin Matuska * Emit or validate (below) a replay record with proper checksums and with
55*d0b3ecdcSMartin Matuska * proper maintenance of the stream checksum. That is:
56*d0b3ecdcSMartin Matuska *
57*d0b3ecdcSMartin Matuska * 1) Update stream checksum with the record header up to drr_checksum.
58*d0b3ecdcSMartin Matuska * 2) Update drr_checksum field in the record header from stream checksum.
59*d0b3ecdcSMartin Matuska * 3) Update stream checksum with the checksum field in the record header.
60*d0b3ecdcSMartin Matuska * 4) Update stream checksum with the contents of the payload.
61*d0b3ecdcSMartin Matuska *
62*d0b3ecdcSMartin Matuska * DRR_BEGIN records do not have record checksums. They can't, because the
63*d0b3ecdcSMartin Matuska * drr_begin struct overlaps with space that would otherwise be used for the
64*d0b3ecdcSMartin Matuska * end-record checksum.
65*d0b3ecdcSMartin Matuska */
66*d0b3ecdcSMartin Matuska static disposition_t
chain_add_fletcher4(drr_packet_t * item,zio_cksum_t * stream_cksum)67*d0b3ecdcSMartin Matuska chain_add_fletcher4(drr_packet_t *item, zio_cksum_t *stream_cksum)
68*d0b3ecdcSMartin Matuska {
69*d0b3ecdcSMartin Matuska if (item == NULL)
70*d0b3ecdcSMartin Matuska return (D_OK);
71*d0b3ecdcSMartin Matuska
72*d0b3ecdcSMartin Matuska dmu_replay_record_t *drr = &item->dp_drr;
73*d0b3ecdcSMartin Matuska struct drr_end *drre = &item->dp_drr.drr_u.drr_end;
74*d0b3ecdcSMartin Matuska zio_cksum_t *record_cksum = &drr->drr_u.drr_checksum.drr_checksum;
75*d0b3ecdcSMartin Matuska zio_cksum_t *end_cksum = &drre->drr_checksum;
76*d0b3ecdcSMartin Matuska
77*d0b3ecdcSMartin Matuska boolean_t swap = OPTION_ENABLED(CA_BYTESWAP_ON_OUTPUT);
78*d0b3ecdcSMartin Matuska uint32_t drr_type = swap ? BSWAP_32(drr->drr_type) : drr->drr_type;
79*d0b3ecdcSMartin Matuska
80*d0b3ecdcSMartin Matuska if (drr_type == DRR_BEGIN) {
81*d0b3ecdcSMartin Matuska ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0);
82*d0b3ecdcSMartin Matuska } else if (drr_type == DRR_END) {
83*d0b3ecdcSMartin Matuska *end_cksum = *stream_cksum;
84*d0b3ecdcSMartin Matuska if (swap)
85*d0b3ecdcSMartin Matuska ZIO_CHECKSUM_BSWAP(end_cksum);
86*d0b3ecdcSMartin Matuska }
87*d0b3ecdcSMartin Matuska fletcher_4_incremental(swap, drr, CK_OFFSET, stream_cksum);
88*d0b3ecdcSMartin Matuska if (drr_type != DRR_BEGIN && !IS_CONCLUSION(drr, drr_type)) {
89*d0b3ecdcSMartin Matuska *record_cksum = *stream_cksum;
90*d0b3ecdcSMartin Matuska if (swap)
91*d0b3ecdcSMartin Matuska ZIO_CHECKSUM_BSWAP(record_cksum);
92*d0b3ecdcSMartin Matuska }
93*d0b3ecdcSMartin Matuska if (drr_type == DRR_END) {
94*d0b3ecdcSMartin Matuska ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0);
95*d0b3ecdcSMartin Matuska } else {
96*d0b3ecdcSMartin Matuska fletcher_4_incremental(swap, record_cksum,
97*d0b3ecdcSMartin Matuska sizeof (drr->drr_u.drr_checksum.drr_checksum),
98*d0b3ecdcSMartin Matuska stream_cksum);
99*d0b3ecdcSMartin Matuska if (item->dp_payload_size > 0) {
100*d0b3ecdcSMartin Matuska fletcher_4_incremental(swap, item->dp_payload,
101*d0b3ecdcSMartin Matuska item->dp_payload_size, stream_cksum);
102*d0b3ecdcSMartin Matuska }
103*d0b3ecdcSMartin Matuska }
104*d0b3ecdcSMartin Matuska return (D_OK);
105*d0b3ecdcSMartin Matuska }
106*d0b3ecdcSMartin Matuska
107*d0b3ecdcSMartin Matuska static disposition_t
chain_validate_fletcher4(drr_packet_t * item,zio_cksum_t * stream_cksum)108*d0b3ecdcSMartin Matuska chain_validate_fletcher4(drr_packet_t *item, zio_cksum_t *stream_cksum)
109*d0b3ecdcSMartin Matuska {
110*d0b3ecdcSMartin Matuska if (item == NULL || OPTION_ENABLED(CA_IGNORE_CKSUMS)) {
111*d0b3ecdcSMartin Matuska return (D_OK);
112*d0b3ecdcSMartin Matuska }
113*d0b3ecdcSMartin Matuska
114*d0b3ecdcSMartin Matuska dmu_replay_record_t *drr = &item->dp_drr;
115*d0b3ecdcSMartin Matuska struct drr_end *drre = &item->dp_drr.drr_u.drr_end;
116*d0b3ecdcSMartin Matuska zio_cksum_t *record_cksum = &drr->drr_u.drr_checksum.drr_checksum;
117*d0b3ecdcSMartin Matuska zio_cksum_t *end_cksum = &drre->drr_checksum;
118*d0b3ecdcSMartin Matuska
119*d0b3ecdcSMartin Matuska boolean_t swap = ATTR_IS_SET(CA_BYTESWAPPED);
120*d0b3ecdcSMartin Matuska uint32_t drr_type = swap ? BSWAP_32(drr->drr_type) : drr->drr_type;
121*d0b3ecdcSMartin Matuska
122*d0b3ecdcSMartin Matuska if (drr_type == DRR_BEGIN) {
123*d0b3ecdcSMartin Matuska ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0);
124*d0b3ecdcSMartin Matuska } else if (drr_type == DRR_END) {
125*d0b3ecdcSMartin Matuska off_t stream_offset = item->dp_stream_offset + END_CK_OFFSET;
126*d0b3ecdcSMartin Matuska validate_or_exit(stream_cksum, end_cksum, swap,
127*d0b3ecdcSMartin Matuska "in DRR_END record", stream_offset);
128*d0b3ecdcSMartin Matuska }
129*d0b3ecdcSMartin Matuska fletcher_4_incremental(swap, drr, CK_OFFSET, stream_cksum);
130*d0b3ecdcSMartin Matuska if (drr_type != DRR_BEGIN && !IS_CONCLUSION(drr, drr_type)) {
131*d0b3ecdcSMartin Matuska off_t stream_offset = item->dp_stream_offset + CK_OFFSET;
132*d0b3ecdcSMartin Matuska validate_or_exit(stream_cksum, record_cksum,
133*d0b3ecdcSMartin Matuska swap, "at DRR record end", stream_offset);
134*d0b3ecdcSMartin Matuska }
135*d0b3ecdcSMartin Matuska if (drr_type == DRR_END) {
136*d0b3ecdcSMartin Matuska ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0);
137*d0b3ecdcSMartin Matuska } else {
138*d0b3ecdcSMartin Matuska fletcher_4_incremental(swap, record_cksum,
139*d0b3ecdcSMartin Matuska sizeof (drr->drr_u.drr_checksum.drr_checksum),
140*d0b3ecdcSMartin Matuska stream_cksum);
141*d0b3ecdcSMartin Matuska if (item->dp_payload_size > 0) {
142*d0b3ecdcSMartin Matuska fletcher_4_incremental(swap, item->dp_payload,
143*d0b3ecdcSMartin Matuska item->dp_payload_size, stream_cksum);
144*d0b3ecdcSMartin Matuska }
145*d0b3ecdcSMartin Matuska }
146*d0b3ecdcSMartin Matuska return (D_OK);
147*d0b3ecdcSMartin Matuska }
148*d0b3ecdcSMartin Matuska
149*d0b3ecdcSMartin Matuska static chain_step_t
fletcher4_serial_step(fletcher4_op_t operation)150*d0b3ecdcSMartin Matuska fletcher4_serial_step(fletcher4_op_t operation)
151*d0b3ecdcSMartin Matuska {
152*d0b3ecdcSMartin Matuska int context_ix = next_context++ % MAX_FLETCHER_4;
153*d0b3ecdcSMartin Matuska fletcher4_context_t *context = &fletcher4_contexts[context_ix];
154*d0b3ecdcSMartin Matuska
155*d0b3ecdcSMartin Matuska ZIO_SET_CHECKSUM(context, 0, 0, 0, 0);
156*d0b3ecdcSMartin Matuska chain_step_t step = {
157*d0b3ecdcSMartin Matuska .cs_type = CS_SERIAL,
158*d0b3ecdcSMartin Matuska .cs_in_size = sizeof (drr_packet_t),
159*d0b3ecdcSMartin Matuska .cs_out_size = sizeof (drr_packet_t),
160*d0b3ecdcSMartin Matuska .cs_context = context,
161*d0b3ecdcSMartin Matuska .cs_serial = {
162*d0b3ecdcSMartin Matuska .process = (zc_serial_process_f *)
163*d0b3ecdcSMartin Matuska ((operation == F4_VALIDATE) ?
164*d0b3ecdcSMartin Matuska chain_validate_fletcher4 : chain_add_fletcher4)
165*d0b3ecdcSMartin Matuska }
166*d0b3ecdcSMartin Matuska };
167*d0b3ecdcSMartin Matuska return (step);
168*d0b3ecdcSMartin Matuska }
169*d0b3ecdcSMartin Matuska
170*d0b3ecdcSMartin Matuska chain_step_t
serial_add_fletcher4(void)171*d0b3ecdcSMartin Matuska serial_add_fletcher4(void)
172*d0b3ecdcSMartin Matuska {
173*d0b3ecdcSMartin Matuska return (fletcher4_serial_step(F4_SET));
174*d0b3ecdcSMartin Matuska }
175*d0b3ecdcSMartin Matuska
176*d0b3ecdcSMartin Matuska chain_step_t
serial_validate_fletcher4(void)177*d0b3ecdcSMartin Matuska serial_validate_fletcher4(void)
178*d0b3ecdcSMartin Matuska {
179*d0b3ecdcSMartin Matuska return (fletcher4_serial_step(F4_VALIDATE));
180*d0b3ecdcSMartin Matuska }
181