1 /* 2 * Copyright (c) Ian F. Darwin 1986-1995. 3 * Software written by Ian F. Darwin and others; 4 * maintained 1995-present by Christos Zoulas and others. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice immediately at the beginning of the file, without modification, 11 * this list of conditions, and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 /* 29 * compress routines: 30 * zmagic() - returns 0 if not recognized, uncompresses and prints 31 * information if recognized 32 * uncompress(method, old, n, newch) - uncompress old into new, 33 * using method, return sizeof new 34 */ 35 #include "file.h" 36 37 #ifndef lint 38 FILE_RCSID("@(#)$File: compress.c,v 1.162 2026/05/17 17:10:25 christos Exp $") 39 #endif 40 41 #include "magic.h" 42 #include <stdlib.h> 43 #ifdef HAVE_UNISTD_H 44 #include <unistd.h> 45 #endif 46 #ifdef HAVE_SPAWN_H 47 #include <spawn.h> 48 #endif 49 #include <stdio.h> 50 #include <string.h> 51 #include <errno.h> 52 #include <ctype.h> 53 #include <stdarg.h> 54 #include <signal.h> 55 #ifndef HAVE_SIG_T 56 typedef void (*sig_t)(int); 57 #endif /* HAVE_SIG_T */ 58 #ifdef HAVE_SYS_IOCTL_H 59 #include <sys/ioctl.h> 60 #endif 61 #ifdef HAVE_SYS_WAIT_H 62 #include <sys/wait.h> 63 #endif 64 #if defined(HAVE_SYS_TIME_H) 65 #include <sys/time.h> 66 #endif 67 68 #if defined(HAVE_ZLIB_H) && defined(ZLIBSUPPORT) 69 #define BUILTIN_DECOMPRESS 70 #include <zlib.h> 71 #endif 72 73 #if defined(HAVE_BZLIB_H) && defined(BZLIBSUPPORT) 74 #define BUILTIN_BZLIB 75 #include <bzlib.h> 76 #endif 77 78 #if defined(HAVE_LZMA_H) && defined(XZLIBSUPPORT) 79 #define BUILTIN_XZLIB 80 #include <lzma.h> 81 #endif 82 83 #if defined(HAVE_ZSTD_H) && defined(ZSTDLIBSUPPORT) 84 #define BUILTIN_ZSTDLIB 85 #include <zstd.h> 86 #include <zstd_errors.h> 87 #endif 88 89 #if defined(HAVE_LZLIB_H) && defined(LZLIBSUPPORT) 90 #define BUILTIN_LZLIB 91 #include <lzlib.h> 92 #endif 93 94 #ifdef notyet 95 #if defined(HAVE_LRZIP_H) && defined(LRZIPLIBSUPPORT) 96 #define BUILTIN_LRZIP 97 #include <Lrzip.h> 98 #endif 99 #endif 100 101 #ifdef DEBUG 102 int tty = -1; 103 #define DPRINTF(...) do { \ 104 if (tty == -1) \ 105 tty = open("/dev/tty", O_RDWR); \ 106 if (tty == -1) \ 107 abort(); \ 108 dprintf(tty, __VA_ARGS__); \ 109 } while (/*CONSTCOND*/0) 110 #else 111 #define DPRINTF(...) 112 #endif 113 114 #ifdef ZLIBSUPPORT 115 /* 116 * The following python code is not really used because ZLIBSUPPORT is only 117 * defined if we have a built-in zlib, and the built-in zlib handles that. 118 * That is not true for android where we have zlib.h and not -lz. 119 */ 120 static const char zlibcode[] = 121 "import sys, zlib; sys.stdout.write(zlib.decompress(sys.stdin.read()))"; 122 123 static const char *zlib_args[] = { "python", "-c", zlibcode, NULL }; 124 125 static int 126 zlibcmp(const unsigned char *buf) 127 { 128 unsigned short x; 129 130 if ((buf[0] & 0xf) != 8 || (buf[0] & 0x80) != 0) 131 return 0; 132 if (file_bigendian()) /* endianness test */ 133 x = buf[0] | (buf[1] << 8); 134 else 135 x = buf[1] | (buf[0] << 8); 136 if (x % 31) 137 return 0; 138 return 1; 139 } 140 #endif 141 142 static int 143 lzmacmp(const unsigned char *buf) 144 { 145 if (buf[0] != 0x5d || buf[1] || buf[2]) 146 return 0; 147 if (buf[12] && buf[12] != 0xff) 148 return 0; 149 return 1; 150 } 151 152 #define gzip_flags "-cd" 153 #define lzip_flags gzip_flags 154 155 static const char *gzip_args[] = { 156 "gzip", gzip_flags, NULL 157 }; 158 static const char *uncompress_args[] = { 159 "uncompress", "-c", NULL 160 }; 161 static const char *bzip2_args[] = { 162 "bzip2", "-cd", NULL 163 }; 164 static const char *lzip_args[] = { 165 "lzip", lzip_flags, NULL 166 }; 167 static const char *xz_args[] = { 168 "xz", "-cd", NULL 169 }; 170 static const char *lrzip_args[] = { 171 "lrzip", "-qdf", "-", NULL 172 }; 173 static const char *lz4_args[] = { 174 "lz4", "-cd", NULL 175 }; 176 static const char *zstd_args[] = { 177 "zstd", "-cd", NULL 178 }; 179 180 #define do_zlib NULL 181 #define do_bzlib NULL 182 183 file_private const struct { 184 union { 185 const char *magic; 186 int (*func)(const unsigned char *); 187 } u; 188 int maglen; 189 const char **argv; 190 void *unused; 191 } compr[] = { 192 #define METH_FROZEN 2 193 #define METH_BZIP 7 194 #define METH_XZ 9 195 #define METH_LZIP 8 196 #define METH_LRZIP 10 197 #define METH_ZSTD 12 198 #define METH_LZMA 13 199 #define METH_ZLIB 14 200 { { .magic = "\037\235" }, 2, gzip_args, NULL }, /* 0, compressed */ 201 /* Uncompress can get stuck; so use gzip first if we have it 202 * Idea from Damien Clark, thanks! */ 203 { { .magic = "\037\235" }, 2, uncompress_args, NULL },/* 1, compressed */ 204 { { .magic = "\037\213" }, 2, gzip_args, do_zlib },/* 2, gzipped */ 205 { { .magic = "\037\236" }, 2, gzip_args, NULL }, /* 3, frozen */ 206 { { .magic = "\037\240" }, 2, gzip_args, NULL }, /* 4, SCO LZH */ 207 /* the standard pack utilities do not accept standard input */ 208 { { .magic = "\037\036" }, 2, gzip_args, NULL }, /* 5, packed */ 209 { { .magic = "PK\3\4" }, 4, gzip_args, NULL }, /* 6, pkziped */ 210 /* ...only first file examined */ 211 { { .magic = "BZh" }, 3, bzip2_args, do_bzlib },/* 7, bzip2-ed */ 212 { { .magic = "LZIP" }, 4, lzip_args, NULL }, /* 8, lzip-ed */ 213 { { .magic = "\3757zXZ\0" },6, xz_args, NULL }, /* 9, XZ Util */ 214 { { .magic = "LRZI" }, 4, lrzip_args, NULL }, /* 10, LRZIP */ 215 { { .magic = "\004\"M\030" },4, lz4_args, NULL }, /* 11, LZ4 */ 216 { { .magic = "\x28\xB5\x2F\xFD" }, 4, zstd_args, NULL },/* 12, zstd */ 217 { { .func = lzmacmp }, -13, xz_args, NULL }, /* 13, lzma */ 218 #ifdef ZLIBSUPPORT 219 { { .func = zlibcmp }, -2, zlib_args, NULL }, /* 14, zlib */ 220 #endif 221 }; 222 223 #define OKDATA 0 224 #define NODATA 1 225 #define ERRDATA 2 226 227 file_private ssize_t swrite(int, const void *, size_t); 228 #if HAVE_FORK 229 file_private size_t ncompr = __arraycount(compr); 230 file_private int uncompressbuf(int, size_t, size_t, int, const unsigned char *, 231 unsigned char **, size_t *); 232 #ifdef BUILTIN_DECOMPRESS 233 file_private int uncompresszlib(const unsigned char *, unsigned char **, size_t, 234 size_t *, int); 235 file_private int uncompressgzipped(const unsigned char *, unsigned char **, size_t, 236 size_t *, int); 237 #endif 238 #ifdef BUILTIN_BZLIB 239 file_private int uncompressbzlib(const unsigned char *, unsigned char **, size_t, 240 size_t *, int); 241 #endif 242 #ifdef BUILTIN_XZLIB 243 file_private int uncompressxzlib(const unsigned char *, unsigned char **, size_t, 244 size_t *, int); 245 #endif 246 #ifdef BUILTIN_ZSTDLIB 247 file_private int uncompresszstd(const unsigned char *, unsigned char **, size_t, 248 size_t *, int); 249 #endif 250 #ifdef BUILTIN_LZLIB 251 file_private int uncompresslzlib(const unsigned char *, unsigned char **, size_t, 252 size_t *, int); 253 #endif 254 #ifdef BUILTIN_LRZIP 255 file_private int uncompresslrzip(const unsigned char *, unsigned char **, size_t, 256 size_t *, int); 257 #endif 258 259 260 static int makeerror(unsigned char **, size_t *, const char *, ...) 261 __attribute__((__format__(__printf__, 3, 4))); 262 file_private const char *methodname(size_t); 263 264 file_private int 265 format_decompression_error(struct magic_set *ms, size_t i, unsigned char *buf) 266 { 267 unsigned char *p; 268 int mime = ms->flags & MAGIC_MIME; 269 270 if (!mime) 271 return file_printf(ms, "ERROR:[%s: %s]", methodname(i), buf); 272 273 for (p = buf; *p; p++) 274 if (!isalnum(*p)) 275 *p = '-'; 276 277 return file_printf(ms, "application/x-decompression-error-%s-%s", 278 methodname(i), buf); 279 } 280 281 file_protected int 282 file_zmagic(struct magic_set *ms, const struct buffer *b, const char *name) 283 { 284 unsigned char *newbuf = NULL; 285 size_t i, nsz; 286 char *rbuf; 287 file_pushbuf_t *pb; 288 int urv, prv, rv = 0; 289 int mime = ms->flags & MAGIC_MIME; 290 int fd = b->fd; 291 const unsigned char *buf = CAST(const unsigned char *, b->fbuf); 292 size_t nbytes = b->flen; 293 int sa_saved = 0; 294 struct sigaction sig_act; 295 296 if ((ms->flags & MAGIC_COMPRESS) == 0) 297 return 0; 298 299 for (i = 0; i < ncompr; i++) { 300 int zm; 301 if (nbytes < CAST(size_t, abs(compr[i].maglen))) 302 continue; 303 if (compr[i].maglen < 0) { 304 zm = (*compr[i].u.func)(buf); 305 } else { 306 zm = memcmp(buf, compr[i].u.magic, 307 CAST(size_t, compr[i].maglen)) == 0; 308 } 309 310 if (!zm) 311 continue; 312 313 /* Prevent SIGPIPE death if child dies unexpectedly */ 314 if (!sa_saved) { 315 //We can use sig_act for both new and old, but 316 struct sigaction new_act; 317 memset(&new_act, 0, sizeof(new_act)); 318 new_act.sa_handler = SIG_IGN; 319 sa_saved = sigaction(SIGPIPE, &new_act, &sig_act) != -1; 320 } 321 322 nsz = nbytes; 323 free(newbuf); 324 urv = uncompressbuf(fd, ms->bytes_max, i, 325 (ms->flags & MAGIC_NO_COMPRESS_FORK), buf, &newbuf, &nsz); 326 DPRINTF("uncompressbuf = %d, %s, %" SIZE_T_FORMAT "u\n", urv, 327 (char *)newbuf, nsz); 328 switch (urv) { 329 case OKDATA: 330 case ERRDATA: 331 ms->flags &= ~MAGIC_COMPRESS; 332 if (urv == ERRDATA) 333 prv = format_decompression_error(ms, i, newbuf); 334 else 335 prv = file_buffer(ms, -1, NULL, name, newbuf, 336 nsz); 337 if (prv == -1) 338 goto error; 339 rv = 1; 340 if ((ms->flags & MAGIC_COMPRESS_TRANSP) != 0) 341 goto out; 342 if (mime != MAGIC_MIME && mime != 0) 343 goto out; 344 if ((file_printf(ms, 345 mime ? " compressed-encoding=" : " (")) == -1) 346 goto error; 347 if ((pb = file_push_buffer(ms)) == NULL) 348 goto error; 349 /* 350 * XXX: If file_buffer fails here, we overwrite 351 * the compressed text. FIXME. 352 */ 353 if (file_buffer(ms, -1, NULL, NULL, buf, nbytes) == -1) 354 { 355 if (file_pop_buffer(ms, pb) != NULL) 356 abort(); 357 goto error; 358 } 359 if ((rbuf = file_pop_buffer(ms, pb)) != NULL) { 360 if (file_printf(ms, "%s", rbuf) == -1) { 361 free(rbuf); 362 goto error; 363 } 364 free(rbuf); 365 } 366 if (!mime && file_printf(ms, ")") == -1) 367 goto error; 368 /*FALLTHROUGH*/ 369 case NODATA: 370 break; 371 default: 372 abort(); 373 /*NOTREACHED*/ 374 error: 375 rv = -1; 376 break; 377 } 378 } 379 out: 380 DPRINTF("rv = %d\n", rv); 381 382 if (sa_saved && sig_act.sa_handler != SIG_IGN) 383 (void)sigaction(SIGPIPE, &sig_act, NULL); 384 385 free(newbuf); 386 ms->flags |= MAGIC_COMPRESS; 387 DPRINTF("Zmagic returns %d\n", rv); 388 return rv; 389 } 390 #endif 391 /* 392 * `safe' write for sockets and pipes. 393 */ 394 file_private ssize_t 395 swrite(int fd, const void *buf, size_t n) 396 { 397 ssize_t rv; 398 size_t rn = n; 399 400 do 401 switch (rv = write(fd, buf, n)) { 402 case -1: 403 if (errno == EINTR) 404 continue; 405 return -1; 406 default: 407 n -= rv; 408 buf = CAST(const char *, buf) + rv; 409 break; 410 } 411 while (n > 0); 412 return rn; 413 } 414 415 416 /* 417 * `safe' read for sockets and pipes. 418 */ 419 file_protected ssize_t 420 sread(int fd, void *buf, size_t n, int canbepipe __attribute__((__unused__))) 421 { 422 ssize_t rv; 423 #if defined(FIONREAD) && !defined(__MINGW32__) 424 int t = 0; 425 #endif 426 size_t rn = n; 427 428 if (fd == STDIN_FILENO) 429 goto nocheck; 430 431 #if defined(FIONREAD) && !defined(__MINGW32__) 432 if (canbepipe && (ioctl(fd, FIONREAD, &t) == -1 || t == 0)) { 433 #ifdef FD_ZERO 434 ssize_t cnt; 435 for (cnt = 0;; cnt++) { 436 fd_set check; 437 struct timeval tout = {0, 100 * 1000}; 438 int selrv; 439 440 FD_ZERO(&check); 441 FD_SET(fd, &check); 442 443 /* 444 * Avoid soft deadlock: do not read if there 445 * is nothing to read from sockets and pipes. 446 */ 447 selrv = select(fd + 1, &check, NULL, NULL, &tout); 448 if (selrv == -1) { 449 if (errno == EINTR || errno == EAGAIN) 450 continue; 451 } else if (selrv == 0 && cnt >= 5) { 452 return 0; 453 } else 454 break; 455 } 456 #endif 457 (void)ioctl(fd, FIONREAD, &t); 458 } 459 460 if (t > 0 && CAST(size_t, t) < n) { 461 n = t; 462 rn = n; 463 } 464 #endif 465 466 nocheck: 467 do 468 switch ((rv = read(fd, buf, n))) { 469 case -1: 470 if (errno == EINTR) 471 continue; 472 return -1; 473 case 0: 474 return rn - n; 475 default: 476 n -= rv; 477 buf = CAST(char *, CCAST(void *, buf)) + rv; 478 break; 479 } 480 while (n > 0); 481 return rn; 482 } 483 484 file_protected int 485 file_pipe2file(struct magic_set *ms, int fd, const void *startbuf, 486 size_t nbytes) 487 { 488 char buf[4096]; 489 ssize_t r; 490 int tfd; 491 492 #ifdef WIN32 493 const char *t; 494 buf[0] = '\0'; 495 if ((t = getenv("TEMP")) != NULL) 496 (void)strlcpy(buf, t, sizeof(buf)); 497 else if ((t = getenv("TMP")) != NULL) 498 (void)strlcpy(buf, t, sizeof(buf)); 499 else if ((t = getenv("TMPDIR")) != NULL) 500 (void)strlcpy(buf, t, sizeof(buf)); 501 if (buf[0] != '\0') 502 (void)strlcat(buf, "/", sizeof(buf)); 503 (void)strlcat(buf, "file.XXXXXX", sizeof(buf)); 504 #else 505 (void)strlcpy(buf, "/tmp/file.XXXXXX", sizeof(buf)); 506 #endif 507 #ifndef HAVE_MKSTEMP 508 { 509 char *ptr = mktemp(buf); 510 tfd = open(ptr, O_RDWR|O_TRUNC|O_EXCL|O_CREAT, 0600); 511 r = errno; 512 (void)unlink(ptr); 513 errno = r; 514 } 515 #else 516 { 517 int te; 518 mode_t ou = umask(0); 519 tfd = mkstemp(buf); 520 (void)umask(ou); 521 te = errno; 522 (void)unlink(buf); 523 errno = te; 524 } 525 #endif 526 if (tfd == -1) { 527 file_error(ms, errno, 528 "cannot create temporary file for pipe copy"); 529 return -1; 530 } 531 532 if (swrite(tfd, startbuf, nbytes) != CAST(ssize_t, nbytes)) 533 r = 1; 534 else { 535 while ((r = sread(fd, buf, sizeof(buf), 1)) > 0) 536 if (swrite(tfd, buf, CAST(size_t, r)) != r) 537 break; 538 } 539 540 switch (r) { 541 case -1: 542 file_error(ms, errno, "error copying from pipe to temp file"); 543 return -1; 544 case 0: 545 break; 546 default: 547 file_error(ms, errno, "error while writing to temp file"); 548 return -1; 549 } 550 551 /* 552 * We duplicate the file descriptor, because fclose on a 553 * tmpfile will delete the file, but any open descriptors 554 * can still access the phantom inode. 555 */ 556 if ((fd = dup2(tfd, fd)) == -1) { 557 file_error(ms, errno, "could not dup descriptor for temp file"); 558 return -1; 559 } 560 (void)close(tfd); 561 if (lseek(fd, CAST(off_t, 0), SEEK_SET) == CAST(off_t, -1)) { 562 file_badseek(ms); 563 return -1; 564 } 565 return fd; 566 } 567 #if HAVE_FORK 568 #ifdef BUILTIN_DECOMPRESS 569 570 #define FHCRC (1 << 1) 571 #define FEXTRA (1 << 2) 572 #define FNAME (1 << 3) 573 #define FCOMMENT (1 << 4) 574 575 576 file_private int 577 uncompressgzipped(const unsigned char *old, unsigned char **newch, 578 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 579 { 580 unsigned char flg; 581 size_t data_start = 10; 582 583 if (*n < 4) { 584 goto err; 585 } 586 587 flg = old[3]; 588 589 if (flg & FEXTRA) { 590 if (data_start + 1 >= *n) 591 goto err; 592 data_start += 2 + old[data_start] + old[data_start + 1] * 256; 593 } 594 if (flg & FNAME) { 595 while(data_start < *n && old[data_start]) 596 data_start++; 597 data_start++; 598 } 599 if (flg & FCOMMENT) { 600 while(data_start < *n && old[data_start]) 601 data_start++; 602 data_start++; 603 } 604 if (flg & FHCRC) 605 data_start += 2; 606 607 if (data_start >= *n) 608 goto err; 609 610 *n -= data_start; 611 old += data_start; 612 return uncompresszlib(old, newch, bytes_max, n, 0); 613 err: 614 return makeerror(newch, n, "File too short"); 615 } 616 617 file_private int 618 uncompresszlib(const unsigned char *old, unsigned char **newch, 619 size_t bytes_max, size_t *n, int zlib) 620 { 621 int rc; 622 z_stream z; 623 624 DPRINTF("builtin zlib decompression\n"); 625 z.next_in = CCAST(Bytef *, old); 626 z.avail_in = CAST(uint32_t, *n); 627 z.next_out = *newch; 628 z.avail_out = CAST(unsigned int, bytes_max); 629 z.zalloc = Z_NULL; 630 z.zfree = Z_NULL; 631 z.opaque = Z_NULL; 632 633 /* LINTED bug in header macro */ 634 rc = zlib ? inflateInit(&z) : inflateInit2(&z, -15); 635 if (rc != Z_OK) 636 goto err; 637 638 rc = inflate(&z, Z_SYNC_FLUSH); 639 if (rc != Z_OK && rc != Z_STREAM_END) { 640 inflateEnd(&z); 641 goto err; 642 } 643 644 *n = CAST(size_t, z.total_out); 645 rc = inflateEnd(&z); 646 if (rc != Z_OK) 647 goto err; 648 649 /* let's keep the nul-terminate tradition */ 650 (*newch)[*n] = '\0'; 651 652 return OKDATA; 653 err: 654 return makeerror(newch, n, "%s", z.msg ? z.msg : zError(rc)); 655 } 656 #endif 657 658 #ifdef BUILTIN_BZLIB 659 file_private int 660 uncompressbzlib(const unsigned char *old, unsigned char **newch, 661 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 662 { 663 int rc; 664 bz_stream bz; 665 666 DPRINTF("builtin bzlib decompression\n"); 667 memset(&bz, 0, sizeof(bz)); 668 rc = BZ2_bzDecompressInit(&bz, 0, 0); 669 if (rc != BZ_OK) 670 goto err; 671 672 bz.next_in = CCAST(char *, RCAST(const char *, old)); 673 bz.avail_in = CAST(uint32_t, *n); 674 bz.next_out = RCAST(char *, *newch); 675 bz.avail_out = CAST(unsigned int, bytes_max); 676 677 rc = BZ2_bzDecompress(&bz); 678 if (rc != BZ_OK && rc != BZ_STREAM_END) { 679 BZ2_bzDecompressEnd(&bz); 680 goto err; 681 } 682 683 /* Assume byte_max is within 32bit */ 684 /* assert(bz.total_out_hi32 == 0); */ 685 *n = CAST(size_t, bz.total_out_lo32); 686 rc = BZ2_bzDecompressEnd(&bz); 687 if (rc != BZ_OK) 688 goto err; 689 690 /* let's keep the nul-terminate tradition */ 691 (*newch)[*n] = '\0'; 692 693 return OKDATA; 694 err: 695 return makeerror(newch, n, "bunzip error %d", rc); 696 } 697 #endif 698 699 #ifdef BUILTIN_XZLIB 700 file_private int 701 uncompressxzlib(const unsigned char *old, unsigned char **newch, 702 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 703 { 704 int rc; 705 lzma_stream xz; 706 707 DPRINTF("builtin xzlib decompression\n"); 708 memset(&xz, 0, sizeof(xz)); 709 rc = lzma_auto_decoder(&xz, UINT64_MAX, 0); 710 if (rc != LZMA_OK) 711 goto err; 712 713 xz.next_in = CCAST(const uint8_t *, old); 714 xz.avail_in = CAST(uint32_t, *n); 715 xz.next_out = RCAST(uint8_t *, *newch); 716 xz.avail_out = CAST(unsigned int, bytes_max); 717 718 rc = lzma_code(&xz, LZMA_RUN); 719 if (rc != LZMA_OK && rc != LZMA_STREAM_END) { 720 lzma_end(&xz); 721 goto err; 722 } 723 724 *n = CAST(size_t, xz.total_out); 725 726 lzma_end(&xz); 727 728 /* let's keep the nul-terminate tradition */ 729 (*newch)[*n] = '\0'; 730 731 return OKDATA; 732 err: 733 return makeerror(newch, n, "unxz error %d", rc); 734 } 735 #endif 736 737 #ifdef BUILTIN_ZSTDLIB 738 file_private int 739 uncompresszstd(const unsigned char *old, unsigned char **newch, 740 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 741 { 742 size_t rc; 743 ZSTD_DStream *zstd; 744 ZSTD_inBuffer in; 745 ZSTD_outBuffer out; 746 747 DPRINTF("builtin zstd decompression\n"); 748 if ((zstd = ZSTD_createDStream()) == NULL) { 749 return makeerror(newch, n, "No ZSTD decompression stream, %s", 750 strerror(errno)); 751 } 752 753 rc = ZSTD_DCtx_reset(zstd, ZSTD_reset_session_only); 754 if (ZSTD_isError(rc)) 755 goto err; 756 757 in.src = CCAST(const void *, old); 758 in.size = *n; 759 in.pos = 0; 760 out.dst = RCAST(void *, *newch); 761 out.size = bytes_max; 762 out.pos = 0; 763 764 rc = ZSTD_decompressStream(zstd, &out, &in); 765 if (ZSTD_isError(rc)) 766 goto err; 767 768 *n = out.pos; 769 770 ZSTD_freeDStream(zstd); 771 772 /* let's keep the nul-terminate tradition */ 773 (*newch)[*n] = '\0'; 774 775 return OKDATA; 776 err: 777 ZSTD_freeDStream(zstd); 778 return makeerror(newch, n, "zstd error %d", ZSTD_getErrorCode(rc)); 779 } 780 #endif 781 782 #ifdef BUILTIN_LZLIB 783 file_private int 784 uncompresslzlib(const unsigned char *old, unsigned char **newch, 785 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 786 { 787 enum LZ_Errno err; 788 size_t old_remaining = *n; 789 size_t new_remaining = bytes_max; 790 size_t total_read = 0; 791 unsigned char *bufp; 792 struct LZ_Decoder *dec; 793 794 bufp = *newch; 795 796 DPRINTF("builtin lzlib decompression\n"); 797 dec = LZ_decompress_open(); 798 if (!dec) { 799 return makeerror(newch, n, "unable to allocate LZ_Decoder"); 800 } 801 if (LZ_decompress_errno(dec) != LZ_ok) 802 goto err; 803 804 for (;;) { 805 // LZ_decompress_read() stops at member boundaries, so we may 806 // have more than one successful read after writing all data 807 // we have. 808 if (old_remaining > 0) { 809 int wr = LZ_decompress_write(dec, old, old_remaining); 810 if (wr < 0) 811 goto err; 812 old_remaining -= wr; 813 old += wr; 814 } 815 816 int rd = LZ_decompress_read(dec, bufp, new_remaining); 817 if (rd > 0) { 818 new_remaining -= rd; 819 bufp += rd; 820 total_read += rd; 821 } 822 823 if (rd < 0 || LZ_decompress_errno(dec) != LZ_ok) 824 goto err; 825 if (new_remaining == 0) 826 break; 827 if (old_remaining == 0 && rd == 0) 828 break; 829 } 830 831 LZ_decompress_close(dec); 832 *n = total_read; 833 834 /* let's keep the nul-terminate tradition */ 835 *bufp = '\0'; 836 837 return OKDATA; 838 err: 839 err = LZ_decompress_errno(dec); 840 LZ_decompress_close(dec); 841 return makeerror(newch, n, "lzlib error: %s", LZ_strerror(err)); 842 } 843 #endif 844 845 #ifdef BUILTIN_LRZIP 846 file_private int 847 uncompresslrzip(const unsigned char *old, unsigned char **newch, 848 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 849 { 850 Lrzip *lr; 851 FILE *in, *out; 852 int res = OKDATA; 853 854 DPRINTF("builtin rlzip decompression\n"); 855 lr = lrzip_new(LRZIP_MODE_DECOMPRESS); 856 if (lr == NULL) { 857 res = makeerror(newch, n, "unable to create an lrzip decoder"); 858 goto out0; 859 } 860 lrzip_config_env(lr); 861 in = fmemopen(RCAST(void *, old), *n, "r"); 862 if (in == NULL) { 863 res = makeerror(newch, n, "unable to construct input file"); 864 goto out1; 865 } 866 if (!lrzip_file_add(lr, in)) { 867 res = makeerror(newch, n, "unable to add input file"); 868 goto out2; 869 } 870 free(*newch); 871 *newch = calloc(*n = 2 * bytes_max, 1); 872 if (*newch == NULL) { 873 res = makeerror(newch, n, "unable to allocate output buffer"); 874 goto out2; 875 } 876 out = fmemopen(*newch, *n, "w"); 877 if (out == NULL) { 878 free(*newch); 879 res = makeerror(newch, n, "unable to allocate output file"); 880 goto out2; 881 } 882 lrzip_outfile_set(lr, out); 883 if (lrzip_run(lr)) { 884 free(*newch); 885 res = makeerror(newch, n, "unable to decompress file"); 886 goto out3; 887 } 888 *n = (size_t)ftell(out); 889 out3: 890 fclose(out); 891 out2: 892 fclose(in); 893 out1: 894 lrzip_free(lr); 895 out0: 896 return res; 897 } 898 #endif 899 900 static int 901 makeerror(unsigned char **buf, size_t *len, const char *fmt, ...) 902 { 903 char *msg; 904 va_list ap; 905 int rv; 906 907 DPRINTF("Makeerror %s\n", fmt); 908 free(*buf); 909 va_start(ap, fmt); 910 rv = vasprintf(&msg, fmt, ap); 911 va_end(ap); 912 if (rv < 0) { 913 DPRINTF("Makeerror failed"); 914 *buf = NULL; 915 *len = 0; 916 return NODATA; 917 } 918 *buf = RCAST(unsigned char *, msg); 919 *len = strlen(msg); 920 return ERRDATA; 921 } 922 923 static void 924 closefd(int *fd, size_t i) 925 { 926 if (fd[i] == -1) 927 return; 928 (void) close(fd[i]); 929 fd[i] = -1; 930 } 931 932 static void 933 closep(int *fd) 934 { 935 size_t i; 936 for (i = 0; i < 2; i++) 937 closefd(fd, i); 938 } 939 940 static void 941 movedesc(void *v, int i, int fd) 942 { 943 if (fd == i) 944 return; /* "no dup was necessary" */ 945 #ifdef HAVE_POSIX_SPAWNP 946 posix_spawn_file_actions_t *fa = RCAST(posix_spawn_file_actions_t *, v); 947 posix_spawn_file_actions_adddup2(fa, fd, i); 948 posix_spawn_file_actions_addclose(fa, fd); 949 #else 950 if (dup2(fd, i) == -1) { 951 DPRINTF("dup(%d, %d) failed (%s)\n", fd, i, strerror(errno)); 952 exit(EXIT_FAILURE); 953 } 954 close(v ? fd : fd); 955 #endif 956 } 957 958 static void 959 closedesc(void *v, int fd) 960 { 961 #ifdef HAVE_POSIX_SPAWNP 962 posix_spawn_file_actions_t *fa = RCAST(posix_spawn_file_actions_t *, v); 963 posix_spawn_file_actions_addclose(fa, fd); 964 #else 965 close(v ? fd : fd); 966 #endif 967 } 968 969 static void 970 handledesc(void *v, int fd, int fdp[3][2]) 971 { 972 if (fd != -1) { 973 (void) lseek(fd, CAST(off_t, 0), SEEK_SET); 974 movedesc(v, STDIN_FILENO, fd); 975 } else { 976 movedesc(v, STDIN_FILENO, fdp[STDIN_FILENO][0]); 977 if (fdp[STDIN_FILENO][1] > 2) 978 closedesc(v, fdp[STDIN_FILENO][1]); 979 } 980 981 file_clear_closexec(STDIN_FILENO); 982 983 ///FIXME: if one of the fdp[i][j] is 0 or 1, this can bomb spectacularly 984 movedesc(v, STDOUT_FILENO, fdp[STDOUT_FILENO][1]); 985 if (fdp[STDOUT_FILENO][0] > 2) 986 closedesc(v, fdp[STDOUT_FILENO][0]); 987 988 file_clear_closexec(STDOUT_FILENO); 989 990 movedesc(v, STDERR_FILENO, fdp[STDERR_FILENO][1]); 991 if (fdp[STDERR_FILENO][0] > 2) 992 closedesc(v, fdp[STDERR_FILENO][0]); 993 994 file_clear_closexec(STDERR_FILENO); 995 } 996 997 static pid_t 998 writechild(int fd, const void *old, size_t n) 999 { 1000 pid_t pid; 1001 1002 /* 1003 * fork again, to avoid blocking because both 1004 * pipes filled 1005 */ 1006 pid = fork(); 1007 if (pid == -1) { 1008 DPRINTF("Fork failed (%s)\n", strerror(errno)); 1009 return -1; 1010 } 1011 if (pid == 0) { 1012 /* child */ 1013 if (swrite(fd, old, n) != CAST(ssize_t, n)) { 1014 DPRINTF("Write failed (%s)\n", strerror(errno)); 1015 exit(EXIT_FAILURE); 1016 } 1017 exit(EXIT_SUCCESS); 1018 } 1019 /* parent */ 1020 return pid; 1021 } 1022 1023 static ssize_t 1024 filter_error(unsigned char *ubuf, ssize_t n) 1025 { 1026 char *p; 1027 char *buf; 1028 1029 ubuf[n] = '\0'; 1030 buf = RCAST(char *, ubuf); 1031 while (isspace(CAST(unsigned char, *buf))) 1032 buf++; 1033 DPRINTF("Filter error[[[%s]]]\n", buf); 1034 if ((p = strchr(CAST(char *, buf), '\n')) != NULL) 1035 *p = '\0'; 1036 if ((p = strchr(CAST(char *, buf), ';')) != NULL) 1037 *p = '\0'; 1038 if ((p = strrchr(CAST(char *, buf), ':')) != NULL) { 1039 ++p; 1040 while (isspace(CAST(unsigned char, *p))) 1041 p++; 1042 n = strlen(p); 1043 memmove(ubuf, p, CAST(size_t, n + 1)); 1044 } 1045 DPRINTF("Filter error after[[[%s]]]\n", (char *)ubuf); 1046 if (islower(*ubuf)) 1047 *ubuf = toupper(*ubuf); 1048 return n; 1049 } 1050 1051 file_private const char * 1052 methodname(size_t method) 1053 { 1054 switch (method) { 1055 #ifdef BUILTIN_DECOMPRESS 1056 case METH_FROZEN: 1057 case METH_ZLIB: 1058 return "zlib"; 1059 #endif 1060 #ifdef BUILTIN_BZLIB 1061 case METH_BZIP: 1062 return "bzlib"; 1063 #endif 1064 #ifdef BUILTIN_XZLIB 1065 case METH_XZ: 1066 case METH_LZMA: 1067 return "xzlib"; 1068 #endif 1069 #ifdef BUILTIN_ZSTDLIB 1070 case METH_ZSTD: 1071 return "zstd"; 1072 #endif 1073 #ifdef BUILTIN_LZLIB 1074 case METH_LZIP: 1075 return "lzlib"; 1076 #endif 1077 #ifdef BUILTIN_LRZIP 1078 case METH_LRZIP: 1079 return "lrzip"; 1080 #endif 1081 default: 1082 return compr[method].argv[0]; 1083 } 1084 } 1085 1086 file_private int (* 1087 getdecompressor(size_t method))(const unsigned char *, unsigned char **, size_t, 1088 size_t *, int) 1089 { 1090 switch (method) { 1091 #ifdef BUILTIN_DECOMPRESS 1092 case METH_FROZEN: 1093 return uncompressgzipped; 1094 case METH_ZLIB: 1095 return uncompresszlib; 1096 #endif 1097 #ifdef BUILTIN_BZLIB 1098 case METH_BZIP: 1099 return uncompressbzlib; 1100 #endif 1101 #ifdef BUILTIN_XZLIB 1102 case METH_XZ: 1103 case METH_LZMA: 1104 return uncompressxzlib; 1105 #endif 1106 #ifdef BUILTIN_ZSTDLIB 1107 case METH_ZSTD: 1108 return uncompresszstd; 1109 #endif 1110 #ifdef BUILTIN_LZLIB 1111 case METH_LZIP: 1112 return uncompresslzlib; 1113 #endif 1114 #ifdef BUILTIN_LRZIP 1115 case METH_LRZIP: 1116 return uncompresslrzip; 1117 #endif 1118 default: 1119 return NULL; 1120 } 1121 } 1122 1123 file_private int 1124 uncompressbuf(int fd, size_t bytes_max, size_t method, int nofork, 1125 const unsigned char *old, unsigned char **newch, size_t* n) 1126 { 1127 int fdp[3][2]; 1128 int status, rv, w; 1129 pid_t pid; 1130 pid_t writepid = -1; 1131 size_t i; 1132 ssize_t r, re; 1133 char *const *args; 1134 #ifdef HAVE_POSIX_SPAWNP 1135 posix_spawn_file_actions_t fa; 1136 #endif 1137 int (*decompress)(const unsigned char *, unsigned char **, 1138 size_t, size_t *, int) = getdecompressor(method); 1139 1140 *newch = CAST(unsigned char *, malloc(bytes_max + 1)); 1141 if (*newch == NULL) 1142 return makeerror(newch, n, "No buffer, %s", strerror(errno)); 1143 1144 if (decompress) { 1145 if (nofork) { 1146 return makeerror(newch, n, 1147 "Fork is required to uncompress, but disabled"); 1148 } 1149 return (*decompress)(old, newch, bytes_max, n, 1); 1150 } 1151 1152 (void)fflush(stdout); 1153 (void)fflush(stderr); 1154 1155 for (i = 0; i < __arraycount(fdp); i++) 1156 fdp[i][0] = fdp[i][1] = -1; 1157 1158 /* 1159 * There are multithreaded users who run magic_file() 1160 * from dozens of threads. If two parallel magic_file() calls 1161 * analyze two large compressed files, both will spawn 1162 * an uncompressing child here, which writes out uncompressed data. 1163 * We read some portion, then close the pipe, then waitpid() the child. 1164 * If uncompressed data is larger, child should get EPIPE and exit. 1165 * However, with *parallel* calls OTHER child may unintentionally 1166 * inherit pipe fds, thus keeping pipe open and making writes in 1167 * our child block instead of failing with EPIPE! 1168 * (For the bug to occur, two threads must mutually inherit their pipes, 1169 * and both must have large outputs. Thus it happens not that often). 1170 * To avoid this, be sure to create pipes with O_CLOEXEC. 1171 */ 1172 if ((fd == -1 && file_pipe_closexec(fdp[STDIN_FILENO]) == -1) || 1173 file_pipe_closexec(fdp[STDOUT_FILENO]) == -1 || 1174 file_pipe_closexec(fdp[STDERR_FILENO]) == -1) { 1175 closep(fdp[STDIN_FILENO]); 1176 closep(fdp[STDOUT_FILENO]); 1177 return makeerror(newch, n, "Cannot create pipe, %s", 1178 strerror(errno)); 1179 } 1180 1181 args = RCAST(char *const *, RCAST(intptr_t, compr[method].argv)); 1182 #ifdef HAVE_POSIX_SPAWNP 1183 posix_spawn_file_actions_init(&fa); 1184 1185 handledesc(&fa, fd, fdp); 1186 1187 DPRINTF("Executing %s\n", compr[method].argv[0]); 1188 status = posix_spawnp(&pid, compr[method].argv[0], &fa, NULL, 1189 args, NULL); 1190 1191 posix_spawn_file_actions_destroy(&fa); 1192 1193 if (status != 0) { 1194 return makeerror(newch, n, "Cannot posix_spawn `%s', %s", 1195 compr[method].argv[0], strerror(status)); 1196 } 1197 #else 1198 /* For processes with large mapped virtual sizes, vfork 1199 * may be _much_ faster (10-100 times) than fork. 1200 */ 1201 pid = vfork(); 1202 if (pid == -1) { 1203 return makeerror(newch, n, "Cannot vfork, %s", 1204 strerror(errno)); 1205 } 1206 if (pid == 0) { 1207 /* child */ 1208 /* Note: we are after vfork, do not modify memory 1209 * in a way which confuses parent. In particular, 1210 * do not modify fdp[i][j]. 1211 */ 1212 handledesc(NULL, fd, fdp); 1213 DPRINTF("Executing %s\n", compr[method].argv[0]); 1214 1215 (void)execvp(compr[method].argv[0], args); 1216 dprintf(STDERR_FILENO, "exec `%s' failed, %s", 1217 compr[method].argv[0], strerror(errno)); 1218 _exit(EXIT_FAILURE); /* _exit(), not exit(), because of vfork */ 1219 } 1220 #endif 1221 /* parent */ 1222 /* Close write sides of child stdout/err pipes */ 1223 for (i = 1; i < __arraycount(fdp); i++) 1224 closefd(fdp[i], 1); 1225 /* Write the buffer data to child stdin, if we don't have fd */ 1226 if (fd == -1) { 1227 closefd(fdp[STDIN_FILENO], 0); 1228 writepid = writechild(fdp[STDIN_FILENO][1], old, *n); 1229 if (writepid == (pid_t)-1) { 1230 rv = makeerror(newch, n, "Write to child failed, %s", 1231 strerror(errno)); 1232 DPRINTF("Write to child failed\n"); 1233 goto err; 1234 } 1235 closefd(fdp[STDIN_FILENO], 1); 1236 } 1237 1238 rv = OKDATA; 1239 r = sread(fdp[STDOUT_FILENO][0], *newch, bytes_max, 0); 1240 DPRINTF("read got %zd\n", r); 1241 if (r < 0) { 1242 rv = ERRDATA; 1243 DPRINTF("Read stdout failed %d (%s)\n", fdp[STDOUT_FILENO][0], 1244 strerror(errno)); 1245 goto err; 1246 } 1247 if (CAST(size_t, r) == bytes_max) { 1248 /* 1249 * close fd so that the child exits with sigpipe and ignore 1250 * errors, otherwise we risk the child blocking and never 1251 * exiting. 1252 */ 1253 DPRINTF("Closing stdout for bytes_max\n"); 1254 closefd(fdp[STDOUT_FILENO], 0); 1255 goto ok; 1256 } 1257 if ((re = sread(fdp[STDERR_FILENO][0], *newch, bytes_max, 0)) > 0) { 1258 DPRINTF("Got stuff from stderr %s\n", *newch); 1259 rv = ERRDATA; 1260 r = filter_error(*newch, r); 1261 goto ok; 1262 } 1263 if (re == 0) 1264 goto ok; 1265 rv = makeerror(newch, n, "Read stderr failed, %s", 1266 strerror(errno)); 1267 goto err; 1268 ok: 1269 *n = r; 1270 /* NUL terminate, as every buffer is handled here. */ 1271 (*newch)[*n] = '\0'; 1272 err: 1273 closefd(fdp[STDIN_FILENO], 1); 1274 closefd(fdp[STDOUT_FILENO], 0); 1275 closefd(fdp[STDERR_FILENO], 0); 1276 1277 w = waitpid(pid, &status, 0); 1278 wait_err: 1279 if (w == -1) { 1280 rv = makeerror(newch, n, "Wait failed, %s", strerror(errno)); 1281 DPRINTF("Child wait return %#x\n", status); 1282 } else if (!WIFEXITED(status)) { 1283 DPRINTF("Child not exited (%#x)\n", status); 1284 } else if (WEXITSTATUS(status) != 0) { 1285 DPRINTF("Child exited (%#x)\n", WEXITSTATUS(status)); 1286 } 1287 if (writepid > 0) { 1288 /* _After_ we know decompressor has exited, our input writer 1289 * definitely will exit now (at worst, writing fails in it, 1290 * since output fd is closed now on the reading size). 1291 */ 1292 w = waitpid(writepid, &status, 0); 1293 writepid = -1; 1294 goto wait_err; 1295 } 1296 1297 closefd(fdp[STDIN_FILENO], 0); //why? it is already closed here! 1298 DPRINTF("Returning %p n=%" SIZE_T_FORMAT "u rv=%d\n", *newch, *n, rv); 1299 1300 return rv; 1301 } 1302 #endif 1303