1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
15 * distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 /*
31 * Subroutines used for writing compressed user process and kernel core dumps.
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_gzio.h"
36 #include "opt_zstdio.h"
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40
41 #include <sys/compressor.h>
42 #include <sys/endian.h>
43 #include <sys/kernel.h>
44 #include <sys/linker_set.h>
45 #include <sys/malloc.h>
46
47 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
48
49 struct compressor_methods {
50 int format;
51 void *(* const init)(size_t, int);
52 void (* const reset)(void *);
53 int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
54 void (* const fini)(void *);
55 };
56
57 struct compressor {
58 const struct compressor_methods *methods;
59 compressor_cb_t cb;
60 void *priv;
61 void *arg;
62 };
63
64 SET_DECLARE(compressors, struct compressor_methods);
65
66 #ifdef GZIO
67
68 #include <contrib/zlib/zutil.h>
69
70 struct gz_stream {
71 uint8_t *gz_buffer; /* output buffer */
72 size_t gz_bufsz; /* output buffer size */
73 off_t gz_off; /* offset into the output stream */
74 uint32_t gz_crc; /* stream CRC32 */
75 z_stream gz_stream; /* zlib state */
76 };
77
78 static void *gz_init(size_t maxiosize, int level);
79 static void gz_reset(void *stream);
80 static int gz_write(void *stream, void *data, size_t len, compressor_cb_t,
81 void *);
82 static void gz_fini(void *stream);
83
84 static void *
gz_alloc(void * arg __unused,u_int n,u_int sz)85 gz_alloc(void *arg __unused, u_int n, u_int sz)
86 {
87
88 /*
89 * Memory for zlib state is allocated using M_NODUMP since it may be
90 * used to compress a kernel dump, and we don't want zlib to attempt to
91 * compress its own state.
92 */
93 return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
94 }
95
96 static void
gz_free(void * arg __unused,void * ptr)97 gz_free(void *arg __unused, void *ptr)
98 {
99
100 free(ptr, M_COMPRESS);
101 }
102
103 static void *
gz_init(size_t maxiosize,int level)104 gz_init(size_t maxiosize, int level)
105 {
106 struct gz_stream *s;
107 int error;
108
109 s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
110 s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
111 s->gz_bufsz = maxiosize;
112
113 s->gz_stream.zalloc = gz_alloc;
114 s->gz_stream.zfree = gz_free;
115 s->gz_stream.opaque = NULL;
116 s->gz_stream.next_in = Z_NULL;
117 s->gz_stream.avail_in = 0;
118
119 if (level != Z_DEFAULT_COMPRESSION) {
120 if (level < Z_BEST_SPEED)
121 level = Z_BEST_SPEED;
122 else if (level > Z_BEST_COMPRESSION)
123 level = Z_BEST_COMPRESSION;
124 }
125
126 error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
127 DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
128 if (error != 0)
129 goto fail;
130
131 gz_reset(s);
132
133 return (s);
134
135 fail:
136 gz_free(NULL, s);
137 return (NULL);
138 }
139
140 static void
gz_reset(void * stream)141 gz_reset(void *stream)
142 {
143 struct gz_stream *s;
144 uint8_t *hdr;
145 const size_t hdrlen = 10;
146
147 s = stream;
148 s->gz_off = 0;
149 s->gz_crc = crc32(0L, Z_NULL, 0);
150
151 (void)deflateReset(&s->gz_stream);
152 s->gz_stream.avail_out = s->gz_bufsz;
153 s->gz_stream.next_out = s->gz_buffer;
154
155 /* Write the gzip header to the output buffer. */
156 hdr = s->gz_buffer;
157 memset(hdr, 0, hdrlen);
158 hdr[0] = 0x1f;
159 hdr[1] = 0x8b;
160 hdr[2] = Z_DEFLATED;
161 hdr[9] = OS_CODE;
162 s->gz_stream.next_out += hdrlen;
163 s->gz_stream.avail_out -= hdrlen;
164 }
165
166 static int
gz_write(void * stream,void * data,size_t len,compressor_cb_t cb,void * arg)167 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
168 void *arg)
169 {
170 struct gz_stream *s;
171 uint8_t trailer[8];
172 size_t room;
173 int error, zerror, zflag;
174
175 s = stream;
176 zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
177
178 if (len > 0) {
179 s->gz_stream.avail_in = len;
180 s->gz_stream.next_in = data;
181 s->gz_crc = crc32(s->gz_crc, data, len);
182 }
183
184 error = 0;
185 do {
186 zerror = deflate(&s->gz_stream, zflag);
187 if (zerror != Z_OK && zerror != Z_STREAM_END) {
188 error = EIO;
189 break;
190 }
191
192 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
193 /*
194 * Our output buffer is full or there's nothing left
195 * to produce, so we're flushing the buffer.
196 */
197 len = s->gz_bufsz - s->gz_stream.avail_out;
198 if (zerror == Z_STREAM_END) {
199 /*
200 * Try to pack as much of the trailer into the
201 * output buffer as we can.
202 */
203 ((uint32_t *)trailer)[0] = htole32(s->gz_crc);
204 ((uint32_t *)trailer)[1] =
205 htole32(s->gz_stream.total_in);
206 room = MIN(sizeof(trailer),
207 s->gz_bufsz - len);
208 memcpy(s->gz_buffer + len, trailer, room);
209 len += room;
210 }
211
212 error = cb(s->gz_buffer, len, s->gz_off, arg);
213 if (error != 0)
214 break;
215
216 s->gz_off += len;
217 s->gz_stream.next_out = s->gz_buffer;
218 s->gz_stream.avail_out = s->gz_bufsz;
219
220 /*
221 * If we couldn't pack the trailer into the output
222 * buffer, write it out now.
223 */
224 if (zerror == Z_STREAM_END && room < sizeof(trailer))
225 error = cb(trailer + room,
226 sizeof(trailer) - room, s->gz_off, arg);
227 }
228 } while (zerror != Z_STREAM_END &&
229 (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
230
231 return (error);
232 }
233
234 static void
gz_fini(void * stream)235 gz_fini(void *stream)
236 {
237 struct gz_stream *s;
238
239 s = stream;
240 (void)deflateEnd(&s->gz_stream);
241 gz_free(NULL, s->gz_buffer);
242 gz_free(NULL, s);
243 }
244
245 struct compressor_methods gzip_methods = {
246 .format = COMPRESS_GZIP,
247 .init = gz_init,
248 .reset = gz_reset,
249 .write = gz_write,
250 .fini = gz_fini,
251 };
252 DATA_SET(compressors, gzip_methods);
253
254 #endif /* GZIO */
255
256 #ifdef ZSTDIO
257
258 #define ZSTD_STATIC_LINKING_ONLY
259 #include <contrib/zstd/lib/zstd.h>
260
261 struct zstdio_stream {
262 ZSTD_CCtx *zst_stream;
263 ZSTD_inBuffer zst_inbuffer;
264 ZSTD_outBuffer zst_outbuffer;
265 uint8_t * zst_buffer; /* output buffer */
266 size_t zst_maxiosz; /* Max output IO size */
267 off_t zst_off; /* offset into the output stream */
268 void * zst_static_wkspc;
269 };
270
271 static void *zstdio_init(size_t maxiosize, int level);
272 static void zstdio_reset(void *stream);
273 static int zstdio_write(void *stream, void *data, size_t len,
274 compressor_cb_t, void *);
275 static void zstdio_fini(void *stream);
276
277 static void *
zstdio_init(size_t maxiosize,int level)278 zstdio_init(size_t maxiosize, int level)
279 {
280 ZSTD_CCtx *dump_compressor;
281 struct zstdio_stream *s;
282 void *wkspc, *owkspc, *buffer;
283 size_t wkspc_size, buf_size, rc;
284
285 s = NULL;
286 wkspc_size = ZSTD_estimateCStreamSize(level);
287 owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
288 M_WAITOK | M_NODUMP);
289 /* Zstd API requires 8-byte alignment. */
290 if ((uintptr_t)wkspc % 8 != 0)
291 wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
292
293 dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
294 if (dump_compressor == NULL) {
295 printf("%s: workspace too small.\n", __func__);
296 goto out;
297 }
298
299 rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
300 if (ZSTD_isError(rc)) {
301 printf("%s: error setting checksumFlag: %s\n", __func__,
302 ZSTD_getErrorName(rc));
303 goto out;
304 }
305 rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
306 level);
307 if (ZSTD_isError(rc)) {
308 printf("%s: error setting compressLevel: %s\n", __func__,
309 ZSTD_getErrorName(rc));
310 goto out;
311 }
312
313 buf_size = ZSTD_CStreamOutSize() * 2;
314 buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
315
316 s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
317 s->zst_buffer = buffer;
318 s->zst_outbuffer.dst = buffer;
319 s->zst_outbuffer.size = buf_size;
320 s->zst_maxiosz = maxiosize;
321 s->zst_stream = dump_compressor;
322 s->zst_static_wkspc = owkspc;
323
324 zstdio_reset(s);
325
326 out:
327 if (s == NULL)
328 free(owkspc, M_COMPRESS);
329 return (s);
330 }
331
332 static void
zstdio_reset(void * stream)333 zstdio_reset(void *stream)
334 {
335 struct zstdio_stream *s;
336 size_t res;
337
338 s = stream;
339 res = ZSTD_CCtx_reset(s->zst_stream, ZSTD_reset_session_only);
340 if (ZSTD_isError(res))
341 panic("%s: could not reset stream %p: %s\n", __func__, s,
342 ZSTD_getErrorName(res));
343 res = ZSTD_CCtx_setPledgedSrcSize(s->zst_stream,
344 ZSTD_CONTENTSIZE_UNKNOWN);
345 if (ZSTD_isError(res))
346 panic("%s: could not set src size on %p: %s\n", __func__, s,
347 ZSTD_getErrorName(res));
348
349 s->zst_off = 0;
350 s->zst_inbuffer.src = NULL;
351 s->zst_inbuffer.size = 0;
352 s->zst_inbuffer.pos = 0;
353 s->zst_outbuffer.pos = 0;
354 }
355
356 static int
zst_flush_intermediate(struct zstdio_stream * s,compressor_cb_t cb,void * arg)357 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
358 {
359 size_t bytes_to_dump;
360 int error;
361
362 /* Flush as many full output blocks as possible. */
363 /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
364 while (s->zst_outbuffer.pos >= 4096) {
365 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
366
367 if (bytes_to_dump > s->zst_maxiosz)
368 bytes_to_dump = s->zst_maxiosz;
369
370 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
371 if (error != 0)
372 return (error);
373
374 /*
375 * Shift any non-full blocks up to the front of the output
376 * buffer.
377 */
378 s->zst_outbuffer.pos -= bytes_to_dump;
379 memmove(s->zst_outbuffer.dst,
380 (char *)s->zst_outbuffer.dst + bytes_to_dump,
381 s->zst_outbuffer.pos);
382 s->zst_off += bytes_to_dump;
383 }
384 return (0);
385 }
386
387 static int
zstdio_flush(struct zstdio_stream * s,compressor_cb_t cb,void * arg)388 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
389 {
390 size_t rc, lastpos;
391 int error;
392
393 /*
394 * Positive return indicates unflushed data remaining; need to call
395 * endStream again after clearing out room in output buffer.
396 */
397 rc = 1;
398 lastpos = s->zst_outbuffer.pos;
399 while (rc > 0) {
400 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
401 if (ZSTD_isError(rc)) {
402 printf("%s: ZSTD_endStream failed (%s)\n", __func__,
403 ZSTD_getErrorName(rc));
404 return (EIO);
405 }
406 if (lastpos == s->zst_outbuffer.pos) {
407 printf("%s: did not make forward progress endStream %zu\n",
408 __func__, lastpos);
409 return (EIO);
410 }
411
412 error = zst_flush_intermediate(s, cb, arg);
413 if (error != 0)
414 return (error);
415
416 lastpos = s->zst_outbuffer.pos;
417 }
418
419 /*
420 * We've already done an intermediate flush, so all full blocks have
421 * been written. Only a partial block remains. Padding happens in a
422 * higher layer.
423 */
424 if (s->zst_outbuffer.pos != 0) {
425 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
426 arg);
427 if (error != 0)
428 return (error);
429 }
430
431 return (0);
432 }
433
434 static int
zstdio_write(void * stream,void * data,size_t len,compressor_cb_t cb,void * arg)435 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
436 void *arg)
437 {
438 struct zstdio_stream *s;
439 size_t lastpos, rc;
440 int error;
441
442 s = stream;
443 if (data == NULL)
444 return (zstdio_flush(s, cb, arg));
445
446 s->zst_inbuffer.src = data;
447 s->zst_inbuffer.size = len;
448 s->zst_inbuffer.pos = 0;
449 lastpos = 0;
450
451 while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
452 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
453 &s->zst_inbuffer);
454 if (ZSTD_isError(rc)) {
455 printf("%s: Compress failed on %p! (%s)\n",
456 __func__, data, ZSTD_getErrorName(rc));
457 return (EIO);
458 }
459
460 if (lastpos == s->zst_inbuffer.pos) {
461 /*
462 * XXX: May need flushStream to make forward progress
463 */
464 printf("ZSTD: did not make forward progress @pos %zu\n",
465 lastpos);
466 return (EIO);
467 }
468 lastpos = s->zst_inbuffer.pos;
469
470 error = zst_flush_intermediate(s, cb, arg);
471 if (error != 0)
472 return (error);
473 }
474 return (0);
475 }
476
477 static void
zstdio_fini(void * stream)478 zstdio_fini(void *stream)
479 {
480 struct zstdio_stream *s;
481
482 s = stream;
483 if (s->zst_static_wkspc != NULL)
484 free(s->zst_static_wkspc, M_COMPRESS);
485 else
486 ZSTD_freeCCtx(s->zst_stream);
487 free(s->zst_buffer, M_COMPRESS);
488 free(s, M_COMPRESS);
489 }
490
491 static struct compressor_methods zstd_methods = {
492 .format = COMPRESS_ZSTD,
493 .init = zstdio_init,
494 .reset = zstdio_reset,
495 .write = zstdio_write,
496 .fini = zstdio_fini,
497 };
498 DATA_SET(compressors, zstd_methods);
499
500 #endif /* ZSTDIO */
501
502 bool
compressor_avail(int format)503 compressor_avail(int format)
504 {
505 struct compressor_methods **iter;
506
507 SET_FOREACH(iter, compressors) {
508 if ((*iter)->format == format)
509 return (true);
510 }
511 return (false);
512 }
513
514 struct compressor *
compressor_init(compressor_cb_t cb,int format,size_t maxiosize,int level,void * arg)515 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
516 void *arg)
517 {
518 struct compressor_methods **iter;
519 struct compressor *s;
520 void *priv;
521
522 SET_FOREACH(iter, compressors) {
523 if ((*iter)->format == format)
524 break;
525 }
526 if (iter == SET_LIMIT(compressors))
527 return (NULL);
528
529 priv = (*iter)->init(maxiosize, level);
530 if (priv == NULL)
531 return (NULL);
532
533 s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
534 s->methods = (*iter);
535 s->priv = priv;
536 s->cb = cb;
537 s->arg = arg;
538 return (s);
539 }
540
541 void
compressor_reset(struct compressor * stream)542 compressor_reset(struct compressor *stream)
543 {
544
545 stream->methods->reset(stream->priv);
546 }
547
548 int
compressor_write(struct compressor * stream,void * data,size_t len)549 compressor_write(struct compressor *stream, void *data, size_t len)
550 {
551
552 return (stream->methods->write(stream->priv, data, len, stream->cb,
553 stream->arg));
554 }
555
556 int
compressor_flush(struct compressor * stream)557 compressor_flush(struct compressor *stream)
558 {
559
560 return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
561 stream->arg));
562 }
563
564 void
compressor_fini(struct compressor * stream)565 compressor_fini(struct compressor *stream)
566 {
567
568 stream->methods->fini(stream->priv);
569 }
570