1 // SPDX-License-Identifier: BSD-3-Clause
2 /*
3 * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the copyright holder nor the names of its
16 * contributors may be used to endorse or promote products derived from this
17 * software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c) 2016-2018, Klara Inc.
34 * Copyright (c) 2016-2018, Allan Jude
35 * Copyright (c) 2018-2020, Sebastian Gottschall
36 * Copyright (c) 2019-2020, Michael Niewöhner
37 * Copyright (c) 2020, The FreeBSD Foundation [1]
38 *
39 * [1] Portions of this software were developed by Allan Jude
40 * under sponsorship from the FreeBSD Foundation.
41 */
42
43 #ifndef _ZFS_ZSTD_H
44 #define _ZFS_ZSTD_H
45
46 #ifdef __cplusplus
47 extern "C" {
48 #endif
49
50 /*
51 * ZSTD block header
52 * NOTE: all fields in this header are in big endian order.
53 */
54 typedef struct zfs_zstd_header {
55 /* Compressed size of data */
56 uint32_t c_len;
57
58 /*
59 * Version and compression level
60 * We used to use a union to reference compression level
61 * and version easily, but as it turns out, relying on the
62 * ordering of bitfields is not remotely portable.
63 * So now we have get/set functions in zfs_zstd.c for
64 * manipulating this in just the right way forever.
65 */
66 uint32_t raw_version_level;
67 char data[];
68 } zfs_zstdhdr_t;
69
70 /*
71 * Simple struct to pass the data from raw_version_level around.
72 */
73 typedef struct zfs_zstd_meta {
74 uint8_t level;
75 uint32_t version;
76 } zfs_zstdmeta_t;
77
78 /*
79 * kstat helper macros
80 */
81 #define ZSTDSTAT(stat) (zstd_stats.stat.value.ui64)
82 #define ZSTDSTAT_ZERO(stat) \
83 atomic_store_64(&zstd_stats.stat.value.ui64, 0)
84 #define ZSTDSTAT_ADD(stat, val) \
85 atomic_add_64(&zstd_stats.stat.value.ui64, (val))
86 #define ZSTDSTAT_SUB(stat, val) \
87 atomic_sub_64(&zstd_stats.stat.value.ui64, (val))
88 #define ZSTDSTAT_BUMP(stat) ZSTDSTAT_ADD(stat, 1)
89
90 /* (de)init for user space / kernel emulation */
91 int zstd_init(void);
92 void zstd_fini(void);
93
94 size_t zfs_zstd_compress(abd_t *src, abd_t *dst, size_t s_len,
95 size_t d_len, int level);
96 int zfs_zstd_get_level(void *s_start, size_t s_len, uint8_t *level);
97 int zfs_zstd_decompress_level(abd_t *src, abd_t *dst, size_t s_len,
98 size_t d_len, uint8_t *level);
99 int zfs_zstd_decompress(abd_t *src, abd_t *dst, size_t s_len,
100 size_t d_len, int n);
101 void zfs_zstd_cache_reap_now(void);
102
103 /*
104 * So, the reason we have all these complicated set/get functions is that
105 * originally, in the zstd "header" we wrote out to disk, we used a 32-bit
106 * bitfield to store the "level" (8 bits) and "version" (24 bits).
107 *
108 * Unfortunately, bitfields make few promises about how they're arranged in
109 * memory...
110 *
111 * By way of example, if we were using version 1.4.5 and level 3, it'd be
112 * level = 0x03, version = 10405/0x0028A5, which gets broken into Vhigh = 0x00,
113 * Vmid = 0x28, Vlow = 0xA5. We include these positions below to help follow
114 * which data winds up where.
115 *
116 * As a consequence, we wound up with little endian platforms with a layout
117 * like this in memory:
118 *
119 * 0 8 16 24 32
120 * +-------+-------+-------+-------+
121 * | Vlow | Vmid | Vhigh | level |
122 * +-------+-------+-------+-------+
123 * =A5 =28 =00 =03
124 *
125 * ...and then, after being run through BE_32(), serializing this out to
126 * disk:
127 *
128 * 0 8 16 24 32
129 * +-------+-------+-------+-------+
130 * | level | Vhigh | Vmid | Vlow |
131 * +-------+-------+-------+-------+
132 * =03 =00 =28 =A5
133 *
134 * while on big-endian systems, since BE_32() is a noop there, both in
135 * memory and on disk, we wind up with:
136 *
137 * 0 8 16 24 32
138 * +-------+-------+-------+-------+
139 * | Vhigh | Vmid | Vlow | level |
140 * +-------+-------+-------+-------+
141 * =00 =28 =A5 =03
142 *
143 * (Vhigh is always 0 until version exceeds 6.55.35. Vmid and Vlow are the
144 * other two bytes of the "version" data.)
145 *
146 * So now we use the BF32_SET macros to get consistent behavior (the
147 * ondisk LE encoding, since x86 currently rules the world) across
148 * platforms, but the "get" behavior requires that we check each of the
149 * bytes in the aforementioned former-bitfield for 0x00, and from there,
150 * we can know which possible layout we're dealing with. (Only the two
151 * that have been observed in the wild are illustrated above, but handlers
152 * for all 4 positions of 0x00 are implemented.
153 */
154
155 static inline void
zfs_get_hdrmeta(const zfs_zstdhdr_t * blob,zfs_zstdmeta_t * res)156 zfs_get_hdrmeta(const zfs_zstdhdr_t *blob, zfs_zstdmeta_t *res)
157 {
158 uint32_t raw = blob->raw_version_level;
159 uint8_t findme = 0xff;
160 int shift;
161 for (shift = 0; shift < 4; shift++) {
162 findme = BF32_GET(raw, 8*shift, 8);
163 if (findme == 0)
164 break;
165 }
166 switch (shift) {
167 case 0:
168 res->level = BF32_GET(raw, 24, 8);
169 res->version = BSWAP_32(raw);
170 res->version = BF32_GET(res->version, 8, 24);
171 break;
172 case 1:
173 res->level = BF32_GET(raw, 0, 8);
174 res->version = BSWAP_32(raw);
175 res->version = BF32_GET(res->version, 0, 24);
176 break;
177 case 2:
178 res->level = BF32_GET(raw, 24, 8);
179 res->version = BF32_GET(raw, 0, 24);
180 break;
181 case 3:
182 res->level = BF32_GET(raw, 0, 8);
183 res->version = BF32_GET(raw, 8, 24);
184 break;
185 default:
186 res->level = 0;
187 res->version = 0;
188 break;
189 }
190 }
191
192 static inline uint8_t
zfs_get_hdrlevel(const zfs_zstdhdr_t * blob)193 zfs_get_hdrlevel(const zfs_zstdhdr_t *blob)
194 {
195 uint8_t level = 0;
196 zfs_zstdmeta_t res;
197 zfs_get_hdrmeta(blob, &res);
198 level = res.level;
199 return (level);
200 }
201
202 static inline uint32_t
zfs_get_hdrversion(const zfs_zstdhdr_t * blob)203 zfs_get_hdrversion(const zfs_zstdhdr_t *blob)
204 {
205 uint32_t version = 0;
206 zfs_zstdmeta_t res;
207 zfs_get_hdrmeta(blob, &res);
208 version = res.version;
209 return (version);
210
211 }
212
213 static inline void
zfs_set_hdrversion(zfs_zstdhdr_t * blob,uint32_t version)214 zfs_set_hdrversion(zfs_zstdhdr_t *blob, uint32_t version)
215 {
216 /* cppcheck-suppress syntaxError */
217 BF32_SET(blob->raw_version_level, 0, 24, version);
218 }
219
220 static inline void
zfs_set_hdrlevel(zfs_zstdhdr_t * blob,uint8_t level)221 zfs_set_hdrlevel(zfs_zstdhdr_t *blob, uint8_t level)
222 {
223 /* cppcheck-suppress syntaxError */
224 BF32_SET(blob->raw_version_level, 24, 8, level);
225 }
226
227
228 #ifdef __cplusplus
229 }
230 #endif
231
232 #endif /* _ZFS_ZSTD_H */
233