1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org> 5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are 9 * met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * Subroutines used for writing compressed user process and kernel core dumps. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_gzio.h" 38 #include "opt_zstdio.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 43 #include <sys/compressor.h> 44 #include <sys/endian.h> 45 #include <sys/kernel.h> 46 #include <sys/linker_set.h> 47 #include <sys/malloc.h> 48 49 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines"); 50 51 struct compressor_methods { 52 int format; 53 void *(* const init)(size_t, int); 54 void (* const reset)(void *); 55 int (* const write)(void *, void *, size_t, compressor_cb_t, void *); 56 void (* const fini)(void *); 57 }; 58 59 struct compressor { 60 const struct compressor_methods *methods; 61 compressor_cb_t cb; 62 void *priv; 63 void *arg; 64 }; 65 66 SET_DECLARE(compressors, struct compressor_methods); 67 68 #ifdef GZIO 69 70 #include <contrib/zlib/zutil.h> 71 72 struct gz_stream { 73 uint8_t *gz_buffer; /* output buffer */ 74 size_t gz_bufsz; /* output buffer size */ 75 off_t gz_off; /* offset into the output stream */ 76 uint32_t gz_crc; /* stream CRC32 */ 77 z_stream gz_stream; /* zlib state */ 78 }; 79 80 static void *gz_init(size_t maxiosize, int level); 81 static void gz_reset(void *stream); 82 static int gz_write(void *stream, void *data, size_t len, compressor_cb_t, 83 void *); 84 static void gz_fini(void *stream); 85 86 static void * 87 gz_alloc(void *arg __unused, u_int n, u_int sz) 88 { 89 90 /* 91 * Memory for zlib state is allocated using M_NODUMP since it may be 92 * used to compress a kernel dump, and we don't want zlib to attempt to 93 * compress its own state. 94 */ 95 return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP)); 96 } 97 98 static void 99 gz_free(void *arg __unused, void *ptr) 100 { 101 102 free(ptr, M_COMPRESS); 103 } 104 105 static void * 106 gz_init(size_t maxiosize, int level) 107 { 108 struct gz_stream *s; 109 int error; 110 111 s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE)); 112 s->gz_buffer = gz_alloc(NULL, 1, maxiosize); 113 s->gz_bufsz = maxiosize; 114 115 s->gz_stream.zalloc = gz_alloc; 116 s->gz_stream.zfree = gz_free; 117 s->gz_stream.opaque = NULL; 118 s->gz_stream.next_in = Z_NULL; 119 s->gz_stream.avail_in = 0; 120 121 if (level != Z_DEFAULT_COMPRESSION) { 122 if (level < Z_BEST_SPEED) 123 level = Z_BEST_SPEED; 124 else if (level > Z_BEST_COMPRESSION) 125 level = Z_BEST_COMPRESSION; 126 } 127 128 error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS, 129 DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); 130 if (error != 0) 131 goto fail; 132 133 gz_reset(s); 134 135 return (s); 136 137 fail: 138 gz_free(NULL, s); 139 return (NULL); 140 } 141 142 static void 143 gz_reset(void *stream) 144 { 145 struct gz_stream *s; 146 uint8_t *hdr; 147 const size_t hdrlen = 10; 148 149 s = stream; 150 s->gz_off = 0; 151 s->gz_crc = crc32(0L, Z_NULL, 0); 152 153 (void)deflateReset(&s->gz_stream); 154 s->gz_stream.avail_out = s->gz_bufsz; 155 s->gz_stream.next_out = s->gz_buffer; 156 157 /* Write the gzip header to the output buffer. */ 158 hdr = s->gz_buffer; 159 memset(hdr, 0, hdrlen); 160 hdr[0] = 0x1f; 161 hdr[1] = 0x8b; 162 hdr[2] = Z_DEFLATED; 163 hdr[9] = OS_CODE; 164 s->gz_stream.next_out += hdrlen; 165 s->gz_stream.avail_out -= hdrlen; 166 } 167 168 static int 169 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb, 170 void *arg) 171 { 172 struct gz_stream *s; 173 uint8_t trailer[8]; 174 size_t room; 175 int error, zerror, zflag; 176 177 s = stream; 178 zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH; 179 180 if (len > 0) { 181 s->gz_stream.avail_in = len; 182 s->gz_stream.next_in = data; 183 s->gz_crc = crc32(s->gz_crc, data, len); 184 } 185 186 error = 0; 187 do { 188 zerror = deflate(&s->gz_stream, zflag); 189 if (zerror != Z_OK && zerror != Z_STREAM_END) { 190 error = EIO; 191 break; 192 } 193 194 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) { 195 /* 196 * Our output buffer is full or there's nothing left 197 * to produce, so we're flushing the buffer. 198 */ 199 len = s->gz_bufsz - s->gz_stream.avail_out; 200 if (zerror == Z_STREAM_END) { 201 /* 202 * Try to pack as much of the trailer into the 203 * output buffer as we can. 204 */ 205 ((uint32_t *)trailer)[0] = htole32(s->gz_crc); 206 ((uint32_t *)trailer)[1] = 207 htole32(s->gz_stream.total_in); 208 room = MIN(sizeof(trailer), 209 s->gz_bufsz - len); 210 memcpy(s->gz_buffer + len, trailer, room); 211 len += room; 212 } 213 214 error = cb(s->gz_buffer, len, s->gz_off, arg); 215 if (error != 0) 216 break; 217 218 s->gz_off += len; 219 s->gz_stream.next_out = s->gz_buffer; 220 s->gz_stream.avail_out = s->gz_bufsz; 221 222 /* 223 * If we couldn't pack the trailer into the output 224 * buffer, write it out now. 225 */ 226 if (zerror == Z_STREAM_END && room < sizeof(trailer)) 227 error = cb(trailer + room, 228 sizeof(trailer) - room, s->gz_off, arg); 229 } 230 } while (zerror != Z_STREAM_END && 231 (zflag == Z_FINISH || s->gz_stream.avail_in > 0)); 232 233 return (error); 234 } 235 236 static void 237 gz_fini(void *stream) 238 { 239 struct gz_stream *s; 240 241 s = stream; 242 (void)deflateEnd(&s->gz_stream); 243 gz_free(NULL, s->gz_buffer); 244 gz_free(NULL, s); 245 } 246 247 struct compressor_methods gzip_methods = { 248 .format = COMPRESS_GZIP, 249 .init = gz_init, 250 .reset = gz_reset, 251 .write = gz_write, 252 .fini = gz_fini, 253 }; 254 DATA_SET(compressors, gzip_methods); 255 256 #endif /* GZIO */ 257 258 #ifdef ZSTDIO 259 260 #define ZSTD_STATIC_LINKING_ONLY 261 #include <contrib/zstd/lib/zstd.h> 262 263 struct zstdio_stream { 264 ZSTD_CCtx *zst_stream; 265 ZSTD_inBuffer zst_inbuffer; 266 ZSTD_outBuffer zst_outbuffer; 267 uint8_t * zst_buffer; /* output buffer */ 268 size_t zst_maxiosz; /* Max output IO size */ 269 off_t zst_off; /* offset into the output stream */ 270 void * zst_static_wkspc; 271 }; 272 273 static void *zstdio_init(size_t maxiosize, int level); 274 static void zstdio_reset(void *stream); 275 static int zstdio_write(void *stream, void *data, size_t len, 276 compressor_cb_t, void *); 277 static void zstdio_fini(void *stream); 278 279 static void * 280 zstdio_init(size_t maxiosize, int level) 281 { 282 ZSTD_CCtx *dump_compressor; 283 struct zstdio_stream *s; 284 void *wkspc, *owkspc, *buffer; 285 size_t wkspc_size, buf_size, rc; 286 287 s = NULL; 288 wkspc_size = ZSTD_estimateCStreamSize(level); 289 owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS, 290 M_WAITOK | M_NODUMP); 291 /* Zstd API requires 8-byte alignment. */ 292 if ((uintptr_t)wkspc % 8 != 0) 293 wkspc = (void *)roundup2((uintptr_t)wkspc, 8); 294 295 dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size); 296 if (dump_compressor == NULL) { 297 printf("%s: workspace too small.\n", __func__); 298 goto out; 299 } 300 301 rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1); 302 if (ZSTD_isError(rc)) { 303 printf("%s: error setting checksumFlag: %s\n", __func__, 304 ZSTD_getErrorName(rc)); 305 goto out; 306 } 307 rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel, 308 level); 309 if (ZSTD_isError(rc)) { 310 printf("%s: error setting compressLevel: %s\n", __func__, 311 ZSTD_getErrorName(rc)); 312 goto out; 313 } 314 315 buf_size = ZSTD_CStreamOutSize() * 2; 316 buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP); 317 318 s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK); 319 s->zst_buffer = buffer; 320 s->zst_outbuffer.dst = buffer; 321 s->zst_outbuffer.size = buf_size; 322 s->zst_maxiosz = maxiosize; 323 s->zst_stream = dump_compressor; 324 s->zst_static_wkspc = owkspc; 325 326 zstdio_reset(s); 327 328 out: 329 if (s == NULL) 330 free(owkspc, M_COMPRESS); 331 return (s); 332 } 333 334 static void 335 zstdio_reset(void *stream) 336 { 337 struct zstdio_stream *s; 338 size_t res; 339 340 s = stream; 341 res = ZSTD_resetCStream(s->zst_stream, 0); 342 if (ZSTD_isError(res)) 343 panic("%s: could not reset stream %p: %s\n", __func__, s, 344 ZSTD_getErrorName(res)); 345 346 s->zst_off = 0; 347 s->zst_inbuffer.src = NULL; 348 s->zst_inbuffer.size = 0; 349 s->zst_inbuffer.pos = 0; 350 s->zst_outbuffer.pos = 0; 351 } 352 353 static int 354 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg) 355 { 356 size_t bytes_to_dump; 357 int error; 358 359 /* Flush as many full output blocks as possible. */ 360 /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */ 361 while (s->zst_outbuffer.pos >= 4096) { 362 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096); 363 364 if (bytes_to_dump > s->zst_maxiosz) 365 bytes_to_dump = s->zst_maxiosz; 366 367 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg); 368 if (error != 0) 369 return (error); 370 371 /* 372 * Shift any non-full blocks up to the front of the output 373 * buffer. 374 */ 375 s->zst_outbuffer.pos -= bytes_to_dump; 376 memmove(s->zst_outbuffer.dst, 377 (char *)s->zst_outbuffer.dst + bytes_to_dump, 378 s->zst_outbuffer.pos); 379 s->zst_off += bytes_to_dump; 380 } 381 return (0); 382 } 383 384 static int 385 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg) 386 { 387 size_t rc, lastpos; 388 int error; 389 390 /* 391 * Positive return indicates unflushed data remaining; need to call 392 * endStream again after clearing out room in output buffer. 393 */ 394 rc = 1; 395 lastpos = s->zst_outbuffer.pos; 396 while (rc > 0) { 397 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer); 398 if (ZSTD_isError(rc)) { 399 printf("%s: ZSTD_endStream failed (%s)\n", __func__, 400 ZSTD_getErrorName(rc)); 401 return (EIO); 402 } 403 if (lastpos == s->zst_outbuffer.pos) { 404 printf("%s: did not make forward progress endStream %zu\n", 405 __func__, lastpos); 406 return (EIO); 407 } 408 409 error = zst_flush_intermediate(s, cb, arg); 410 if (error != 0) 411 return (error); 412 413 lastpos = s->zst_outbuffer.pos; 414 } 415 416 /* 417 * We've already done an intermediate flush, so all full blocks have 418 * been written. Only a partial block remains. Padding happens in a 419 * higher layer. 420 */ 421 if (s->zst_outbuffer.pos != 0) { 422 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off, 423 arg); 424 if (error != 0) 425 return (error); 426 } 427 428 return (0); 429 } 430 431 static int 432 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb, 433 void *arg) 434 { 435 struct zstdio_stream *s; 436 size_t lastpos, rc; 437 int error; 438 439 s = stream; 440 if (data == NULL) 441 return (zstdio_flush(s, cb, arg)); 442 443 s->zst_inbuffer.src = data; 444 s->zst_inbuffer.size = len; 445 s->zst_inbuffer.pos = 0; 446 lastpos = 0; 447 448 while (s->zst_inbuffer.pos < s->zst_inbuffer.size) { 449 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer, 450 &s->zst_inbuffer); 451 if (ZSTD_isError(rc)) { 452 printf("%s: Compress failed on %p! (%s)\n", 453 __func__, data, ZSTD_getErrorName(rc)); 454 return (EIO); 455 } 456 457 if (lastpos == s->zst_inbuffer.pos) { 458 /* 459 * XXX: May need flushStream to make forward progress 460 */ 461 printf("ZSTD: did not make forward progress @pos %zu\n", 462 lastpos); 463 return (EIO); 464 } 465 lastpos = s->zst_inbuffer.pos; 466 467 error = zst_flush_intermediate(s, cb, arg); 468 if (error != 0) 469 return (error); 470 } 471 return (0); 472 } 473 474 static void 475 zstdio_fini(void *stream) 476 { 477 struct zstdio_stream *s; 478 479 s = stream; 480 if (s->zst_static_wkspc != NULL) 481 free(s->zst_static_wkspc, M_COMPRESS); 482 else 483 ZSTD_freeCCtx(s->zst_stream); 484 free(s->zst_buffer, M_COMPRESS); 485 free(s, M_COMPRESS); 486 } 487 488 static struct compressor_methods zstd_methods = { 489 .format = COMPRESS_ZSTD, 490 .init = zstdio_init, 491 .reset = zstdio_reset, 492 .write = zstdio_write, 493 .fini = zstdio_fini, 494 }; 495 DATA_SET(compressors, zstd_methods); 496 497 #endif /* ZSTDIO */ 498 499 bool 500 compressor_avail(int format) 501 { 502 struct compressor_methods **iter; 503 504 SET_FOREACH(iter, compressors) { 505 if ((*iter)->format == format) 506 return (true); 507 } 508 return (false); 509 } 510 511 struct compressor * 512 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level, 513 void *arg) 514 { 515 struct compressor_methods **iter; 516 struct compressor *s; 517 void *priv; 518 519 SET_FOREACH(iter, compressors) { 520 if ((*iter)->format == format) 521 break; 522 } 523 if (iter == SET_LIMIT(compressors)) 524 return (NULL); 525 526 priv = (*iter)->init(maxiosize, level); 527 if (priv == NULL) 528 return (NULL); 529 530 s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO); 531 s->methods = (*iter); 532 s->priv = priv; 533 s->cb = cb; 534 s->arg = arg; 535 return (s); 536 } 537 538 void 539 compressor_reset(struct compressor *stream) 540 { 541 542 stream->methods->reset(stream->priv); 543 } 544 545 int 546 compressor_write(struct compressor *stream, void *data, size_t len) 547 { 548 549 return (stream->methods->write(stream->priv, data, len, stream->cb, 550 stream->arg)); 551 } 552 553 int 554 compressor_flush(struct compressor *stream) 555 { 556 557 return (stream->methods->write(stream->priv, NULL, 0, stream->cb, 558 stream->arg)); 559 } 560 561 void 562 compressor_fini(struct compressor *stream) 563 { 564 565 stream->methods->fini(stream->priv); 566 } 567