1 /* 2 * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /* ************************************* 13 * Compiler Options 14 ***************************************/ 15 #ifdef _MSC_VER /* Visual */ 16 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 17 # pragma warning(disable : 4204) /* non-constant aggregate initializer */ 18 #endif 19 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) 20 # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ 21 #endif 22 23 /*-************************************* 24 * Includes 25 ***************************************/ 26 #include "platform.h" /* Large Files support, SET_BINARY_MODE */ 27 #include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ 28 #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ 29 #include <stdlib.h> /* malloc, free */ 30 #include <string.h> /* strcmp, strlen */ 31 #include <assert.h> 32 #include <errno.h> /* errno */ 33 #include <limits.h> /* INT_MAX */ 34 #include <signal.h> 35 #include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ 36 37 #if defined (_MSC_VER) 38 # include <sys/stat.h> 39 # include <io.h> 40 #endif 41 42 #include "../lib/common/mem.h" /* U32, U64 */ 43 #include "fileio.h" 44 45 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ 46 #include "../lib/zstd.h" 47 #include "../lib/common/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */ 48 #include "../lib/compress/zstd_compress_internal.h" 49 50 #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) 51 # include <zlib.h> 52 # if !defined(z_const) 53 # define z_const 54 # endif 55 #endif 56 57 #if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) 58 # include <lzma.h> 59 #endif 60 61 #define LZ4_MAGICNUMBER 0x184D2204 62 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) 63 # define LZ4F_ENABLE_OBSOLETE_ENUMS 64 # include <lz4frame.h> 65 # include <lz4.h> 66 #endif 67 68 69 /*-************************************* 70 * Constants 71 ***************************************/ 72 #define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ 73 #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ 74 75 #define FNSPACE 30 76 77 /*-************************************* 78 * Macros 79 ***************************************/ 80 81 struct FIO_display_prefs_s { 82 int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ 83 U32 noProgress; 84 }; 85 86 static FIO_display_prefs_t g_display_prefs = {2, 0}; 87 88 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 89 #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) 90 #define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } 91 92 static const U64 g_refreshRate = SEC_TO_MICRO / 6; 93 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; 94 95 #define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) 96 #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } 97 #define DISPLAYUPDATE(l, ...) { \ 98 if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \ 99 if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ 100 DELAY_NEXT_UPDATE(); \ 101 DISPLAY(__VA_ARGS__); \ 102 if (g_display_prefs.displayLevel>=4) fflush(stderr); \ 103 } } } 104 105 #undef MIN /* in case it would be already defined */ 106 #define MIN(a,b) ((a) < (b) ? (a) : (b)) 107 108 109 #define EXM_THROW(error, ...) \ 110 { \ 111 DISPLAYLEVEL(1, "zstd: "); \ 112 DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 113 DISPLAYLEVEL(1, "error %i : ", error); \ 114 DISPLAYLEVEL(1, __VA_ARGS__); \ 115 DISPLAYLEVEL(1, " \n"); \ 116 exit(error); \ 117 } 118 119 #define CHECK_V(v, f) \ 120 v = f; \ 121 if (ZSTD_isError(v)) { \ 122 DISPLAYLEVEL(5, "%s \n", #f); \ 123 EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ 124 } 125 #define CHECK(f) { size_t err; CHECK_V(err, f); } 126 127 128 /*-************************************ 129 * Signal (Ctrl-C trapping) 130 **************************************/ 131 static const char* g_artefact = NULL; 132 static void INThandler(int sig) 133 { 134 assert(sig==SIGINT); (void)sig; 135 #if !defined(_MSC_VER) 136 signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ 137 #endif 138 if (g_artefact) { 139 assert(UTIL_isRegularFile(g_artefact)); 140 remove(g_artefact); 141 } 142 DISPLAY("\n"); 143 exit(2); 144 } 145 static void addHandler(char const* dstFileName) 146 { 147 if (UTIL_isRegularFile(dstFileName)) { 148 g_artefact = dstFileName; 149 signal(SIGINT, INThandler); 150 } else { 151 g_artefact = NULL; 152 } 153 } 154 /* Idempotent */ 155 static void clearHandler(void) 156 { 157 if (g_artefact) signal(SIGINT, SIG_DFL); 158 g_artefact = NULL; 159 } 160 161 162 /*-********************************************************* 163 * Termination signal trapping (Print debug stack trace) 164 ***********************************************************/ 165 #if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */ 166 # if (__has_feature(address_sanitizer)) 167 # define BACKTRACE_ENABLE 0 168 # endif /* __has_feature(address_sanitizer) */ 169 #elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */ 170 # define BACKTRACE_ENABLE 0 171 #endif 172 173 #if !defined(BACKTRACE_ENABLE) 174 /* automatic detector : backtrace enabled by default on linux+glibc and osx */ 175 # if (defined(__linux__) && (defined(__GLIBC__) && !defined(__UCLIBC__))) \ 176 || (defined(__APPLE__) && defined(__MACH__)) 177 # define BACKTRACE_ENABLE 1 178 # else 179 # define BACKTRACE_ENABLE 0 180 # endif 181 #endif 182 183 /* note : after this point, BACKTRACE_ENABLE is necessarily defined */ 184 185 186 #if BACKTRACE_ENABLE 187 188 #include <execinfo.h> /* backtrace, backtrace_symbols */ 189 190 #define MAX_STACK_FRAMES 50 191 192 static void ABRThandler(int sig) { 193 const char* name; 194 void* addrlist[MAX_STACK_FRAMES]; 195 char** symbollist; 196 int addrlen, i; 197 198 switch (sig) { 199 case SIGABRT: name = "SIGABRT"; break; 200 case SIGFPE: name = "SIGFPE"; break; 201 case SIGILL: name = "SIGILL"; break; 202 case SIGINT: name = "SIGINT"; break; 203 case SIGSEGV: name = "SIGSEGV"; break; 204 default: name = "UNKNOWN"; 205 } 206 207 DISPLAY("Caught %s signal, printing stack:\n", name); 208 /* Retrieve current stack addresses. */ 209 addrlen = backtrace(addrlist, MAX_STACK_FRAMES); 210 if (addrlen == 0) { 211 DISPLAY("\n"); 212 return; 213 } 214 /* Create readable strings to each frame. */ 215 symbollist = backtrace_symbols(addrlist, addrlen); 216 /* Print the stack trace, excluding calls handling the signal. */ 217 for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) { 218 DISPLAY("%s\n", symbollist[i]); 219 } 220 free(symbollist); 221 /* Reset and raise the signal so default handler runs. */ 222 signal(sig, SIG_DFL); 223 raise(sig); 224 } 225 #endif 226 227 void FIO_addAbortHandler() 228 { 229 #if BACKTRACE_ENABLE 230 signal(SIGABRT, ABRThandler); 231 signal(SIGFPE, ABRThandler); 232 signal(SIGILL, ABRThandler); 233 signal(SIGSEGV, ABRThandler); 234 signal(SIGBUS, ABRThandler); 235 #endif 236 } 237 238 239 /*-************************************************************ 240 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW 241 ***************************************************************/ 242 #if defined(_MSC_VER) && _MSC_VER >= 1400 243 # define LONG_SEEK _fseeki64 244 # define LONG_TELL _ftelli64 245 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 246 # define LONG_SEEK fseeko 247 # define LONG_TELL ftello 248 #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) 249 # define LONG_SEEK fseeko64 250 # define LONG_TELL ftello64 251 #elif defined(_WIN32) && !defined(__DJGPP__) 252 # include <windows.h> 253 static int LONG_SEEK(FILE* file, __int64 offset, int origin) { 254 LARGE_INTEGER off; 255 DWORD method; 256 off.QuadPart = offset; 257 if (origin == SEEK_END) 258 method = FILE_END; 259 else if (origin == SEEK_CUR) 260 method = FILE_CURRENT; 261 else 262 method = FILE_BEGIN; 263 264 if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) 265 return 0; 266 else 267 return -1; 268 } 269 static __int64 LONG_TELL(FILE* file) { 270 LARGE_INTEGER off, newOff; 271 off.QuadPart = 0; 272 newOff.QuadPart = 0; 273 SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); 274 return newOff.QuadPart; 275 } 276 #else 277 # define LONG_SEEK fseek 278 # define LONG_TELL ftell 279 #endif 280 281 282 /*-************************************* 283 * Parameters: FIO_prefs_t 284 ***************************************/ 285 286 /* typedef'd to FIO_prefs_t within fileio.h */ 287 struct FIO_prefs_s { 288 289 /* Algorithm preferences */ 290 FIO_compressionType_t compressionType; 291 U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ 292 int dictIDFlag; 293 int checksumFlag; 294 int blockSize; 295 int overlapLog; 296 U32 adaptiveMode; 297 int rsyncable; 298 int minAdaptLevel; 299 int maxAdaptLevel; 300 int ldmFlag; 301 int ldmHashLog; 302 int ldmMinMatch; 303 int ldmBucketSizeLog; 304 int ldmHashRateLog; 305 size_t streamSrcSize; 306 size_t targetCBlockSize; 307 int srcSizeHint; 308 int testMode; 309 ZSTD_literalCompressionMode_e literalCompressionMode; 310 311 /* IO preferences */ 312 U32 removeSrcFile; 313 U32 overwrite; 314 315 /* Computation resources preferences */ 316 unsigned memLimit; 317 int nbWorkers; 318 319 int excludeCompressedFiles; 320 int patchFromMode; 321 int contentSize; 322 }; 323 324 325 /*-************************************* 326 * Parameters: Initialization 327 ***************************************/ 328 329 #define FIO_OVERLAP_LOG_NOTSET 9999 330 #define FIO_LDM_PARAM_NOTSET 9999 331 332 333 FIO_prefs_t* FIO_createPreferences(void) 334 { 335 FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t)); 336 if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); 337 338 ret->compressionType = FIO_zstdCompression; 339 ret->overwrite = 0; 340 ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; 341 ret->dictIDFlag = 1; 342 ret->checksumFlag = 1; 343 ret->removeSrcFile = 0; 344 ret->memLimit = 0; 345 ret->nbWorkers = 1; 346 ret->blockSize = 0; 347 ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; 348 ret->adaptiveMode = 0; 349 ret->rsyncable = 0; 350 ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ 351 ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ 352 ret->ldmFlag = 0; 353 ret->ldmHashLog = 0; 354 ret->ldmMinMatch = 0; 355 ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; 356 ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; 357 ret->streamSrcSize = 0; 358 ret->targetCBlockSize = 0; 359 ret->srcSizeHint = 0; 360 ret->testMode = 0; 361 ret->literalCompressionMode = ZSTD_lcm_auto; 362 ret->excludeCompressedFiles = 0; 363 return ret; 364 } 365 366 void FIO_freePreferences(FIO_prefs_t* const prefs) 367 { 368 free(prefs); 369 } 370 371 372 /*-************************************* 373 * Parameters: Display Options 374 ***************************************/ 375 376 void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } 377 378 void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; } 379 380 381 /*-************************************* 382 * Parameters: Setters 383 ***************************************/ 384 385 void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } 386 387 void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } 388 389 void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } 390 391 void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } 392 393 void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; } 394 395 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); } 396 397 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } 398 399 void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { 400 #ifndef ZSTD_MULTITHREAD 401 if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); 402 #endif 403 prefs->nbWorkers = nbWorkers; 404 } 405 406 void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } 407 408 void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { 409 if (blockSize && prefs->nbWorkers==0) 410 DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); 411 prefs->blockSize = blockSize; 412 } 413 414 void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ 415 if (overlapLog && prefs->nbWorkers==0) 416 DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); 417 prefs->overlapLog = overlapLog; 418 } 419 420 void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) { 421 if ((adapt>0) && (prefs->nbWorkers==0)) 422 EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); 423 prefs->adaptiveMode = adapt; 424 } 425 426 void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { 427 if ((rsyncable>0) && (prefs->nbWorkers==0)) 428 EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); 429 prefs->rsyncable = rsyncable; 430 } 431 432 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) { 433 prefs->streamSrcSize = streamSrcSize; 434 } 435 436 void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) { 437 prefs->targetCBlockSize = targetCBlockSize; 438 } 439 440 void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { 441 prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); 442 } 443 444 void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { 445 prefs->testMode = (testMode!=0); 446 } 447 448 void FIO_setLiteralCompressionMode( 449 FIO_prefs_t* const prefs, 450 ZSTD_literalCompressionMode_e mode) { 451 prefs->literalCompressionMode = mode; 452 } 453 454 void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) 455 { 456 #ifndef ZSTD_NOCOMPRESS 457 assert(minCLevel >= ZSTD_minCLevel()); 458 #endif 459 prefs->minAdaptLevel = minCLevel; 460 } 461 462 void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel) 463 { 464 prefs->maxAdaptLevel = maxCLevel; 465 } 466 467 void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) { 468 prefs->ldmFlag = (ldmFlag>0); 469 } 470 471 void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) { 472 prefs->ldmHashLog = ldmHashLog; 473 } 474 475 void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) { 476 prefs->ldmMinMatch = ldmMinMatch; 477 } 478 479 void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) { 480 prefs->ldmBucketSizeLog = ldmBucketSizeLog; 481 } 482 483 484 void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) { 485 prefs->ldmHashRateLog = ldmHashRateLog; 486 } 487 488 void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value) 489 { 490 prefs->patchFromMode = value != 0; 491 } 492 493 void FIO_setContentSize(FIO_prefs_t* const prefs, int value) 494 { 495 prefs->contentSize = value != 0; 496 } 497 498 /*-************************************* 499 * Functions 500 ***************************************/ 501 /** FIO_remove() : 502 * @result : Unlink `fileName`, even if it's read-only */ 503 static int FIO_remove(const char* path) 504 { 505 if (!UTIL_isRegularFile(path)) { 506 DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s \n", path); 507 return 0; 508 } 509 #if defined(_WIN32) || defined(WIN32) 510 /* windows doesn't allow remove read-only files, 511 * so try to make it writable first */ 512 UTIL_chmod(path, _S_IWRITE); 513 #endif 514 return remove(path); 515 } 516 517 /** FIO_openSrcFile() : 518 * condition : `srcFileName` must be non-NULL. 519 * @result : FILE* to `srcFileName`, or NULL if it fails */ 520 static FILE* FIO_openSrcFile(const char* srcFileName) 521 { 522 assert(srcFileName != NULL); 523 if (!strcmp (srcFileName, stdinmark)) { 524 DISPLAYLEVEL(4,"Using stdin for input \n"); 525 SET_BINARY_MODE(stdin); 526 return stdin; 527 } 528 529 if (!UTIL_fileExist(srcFileName)) { 530 DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", 531 srcFileName, strerror(errno)); 532 return NULL; 533 } 534 535 if (!UTIL_isRegularFile(srcFileName) 536 && !UTIL_isFIFO(srcFileName) 537 ) { 538 DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", 539 srcFileName); 540 return NULL; 541 } 542 543 { FILE* const f = fopen(srcFileName, "rb"); 544 if (f == NULL) 545 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 546 return f; 547 } 548 } 549 550 /** FIO_openDstFile() : 551 * condition : `dstFileName` must be non-NULL. 552 * @result : FILE* to `dstFileName`, or NULL if it fails */ 553 static FILE* 554 FIO_openDstFile(FIO_prefs_t* const prefs, 555 const char* srcFileName, const char* dstFileName) 556 { 557 if (prefs->testMode) return NULL; /* do not open file in test mode */ 558 559 assert(dstFileName != NULL); 560 if (!strcmp (dstFileName, stdoutmark)) { 561 DISPLAYLEVEL(4,"Using stdout for output \n"); 562 SET_BINARY_MODE(stdout); 563 if (prefs->sparseFileSupport == 1) { 564 prefs->sparseFileSupport = 0; 565 DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 566 } 567 return stdout; 568 } 569 570 /* ensure dst is not the same as src */ 571 if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { 572 DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); 573 return NULL; 574 } 575 576 if (prefs->sparseFileSupport == 1) { 577 prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT; 578 } 579 580 if (UTIL_isRegularFile(dstFileName)) { 581 /* Check if destination file already exists */ 582 FILE* const fCheck = fopen( dstFileName, "rb" ); 583 #if !defined(_WIN32) 584 /* this test does not work on Windows : 585 * `NUL` and `nul` are detected as regular files */ 586 if (!strcmp(dstFileName, nulmark)) { 587 EXM_THROW(40, "%s is unexpectedly categorized as a regular file", 588 dstFileName); 589 } 590 #endif 591 if (fCheck != NULL) { /* dst file exists, authorization prompt */ 592 fclose(fCheck); 593 if (!prefs->overwrite) { 594 if (g_display_prefs.displayLevel <= 1) { 595 /* No interaction possible */ 596 DISPLAY("zstd: %s already exists; not overwritten \n", 597 dstFileName); 598 return NULL; 599 } 600 DISPLAY("zstd: %s already exists; overwrite (y/N) ? ", 601 dstFileName); 602 { int ch = getchar(); 603 if ((ch!='Y') && (ch!='y')) { 604 DISPLAY(" not overwritten \n"); 605 return NULL; 606 } 607 /* flush rest of input line */ 608 while ((ch!=EOF) && (ch!='\n')) ch = getchar(); 609 } } 610 /* need to unlink */ 611 FIO_remove(dstFileName); 612 } } 613 614 { FILE* const f = fopen( dstFileName, "wb" ); 615 if (f == NULL) { 616 DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); 617 } else if (srcFileName != NULL 618 && strcmp (srcFileName, stdinmark) 619 && strcmp(dstFileName, nulmark) ) { 620 /* reduce rights on newly created dst file while compression is ongoing */ 621 UTIL_chmod(dstFileName, 00600); 622 } 623 return f; 624 } 625 } 626 627 628 /*! FIO_createDictBuffer() : 629 * creates a buffer, pointed by `*bufferPtr`, 630 * loads `filename` content into it, up to DICTSIZE_MAX bytes. 631 * @return : loaded size 632 * if fileName==NULL, returns 0 and a NULL pointer 633 */ 634 static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs) 635 { 636 FILE* fileHandle; 637 U64 fileSize; 638 639 assert(bufferPtr != NULL); 640 *bufferPtr = NULL; 641 if (fileName == NULL) return 0; 642 643 DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); 644 fileHandle = fopen(fileName, "rb"); 645 if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); 646 647 fileSize = UTIL_getFileSize(fileName); 648 { 649 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; 650 if (fileSize > dictSizeMax) { 651 EXM_THROW(32, "Dictionary file %s is too large (> %u bytes)", 652 fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ 653 } 654 } 655 *bufferPtr = malloc((size_t)fileSize); 656 if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); 657 { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); 658 if (readSize != fileSize) 659 EXM_THROW(35, "Error reading dictionary file %s : %s", 660 fileName, strerror(errno)); 661 } 662 fclose(fileHandle); 663 return (size_t)fileSize; 664 } 665 666 667 668 /* FIO_checkFilenameCollisions() : 669 * Checks for and warns if there are any files that would have the same output path 670 */ 671 int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { 672 const char **filenameTableSorted, *c, *prevElem, *filename; 673 unsigned u; 674 675 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ 676 c = "\\"; 677 #else 678 c = "/"; 679 #endif 680 681 filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); 682 if (!filenameTableSorted) { 683 DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); 684 return 1; 685 } 686 687 for (u = 0; u < nbFiles; ++u) { 688 filename = strrchr(filenameTable[u], c[0]); 689 if (filename == NULL) { 690 filenameTableSorted[u] = filenameTable[u]; 691 } else { 692 filenameTableSorted[u] = filename+1; 693 } 694 } 695 696 qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); 697 prevElem = filenameTableSorted[0]; 698 for (u = 1; u < nbFiles; ++u) { 699 if (strcmp(prevElem, filenameTableSorted[u]) == 0) { 700 DISPLAY("WARNING: Two files have same filename: %s\n", prevElem); 701 } 702 prevElem = filenameTableSorted[u]; 703 } 704 705 free((void*)filenameTableSorted); 706 return 0; 707 } 708 709 static const char* 710 extractFilename(const char* path, char separator) 711 { 712 const char* search = strrchr(path, separator); 713 if (search == NULL) return path; 714 return search+1; 715 } 716 717 /* FIO_createFilename_fromOutDir() : 718 * Takes a source file name and specified output directory, and 719 * allocates memory for and returns a pointer to final path. 720 * This function never returns an error (it may abort() in case of pb) 721 */ 722 static char* 723 FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen) 724 { 725 const char* filenameStart; 726 char separator; 727 char* result; 728 729 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ 730 separator = '\\'; 731 #else 732 separator = '/'; 733 #endif 734 735 filenameStart = extractFilename(path, separator); 736 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ 737 filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */ 738 #endif 739 740 result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1); 741 if (!result) { 742 EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno)); 743 } 744 745 memcpy(result, outDirName, strlen(outDirName)); 746 if (outDirName[strlen(outDirName)-1] == separator) { 747 memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart)); 748 } else { 749 memcpy(result + strlen(outDirName), &separator, 1); 750 memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart)); 751 } 752 753 return result; 754 } 755 756 /* FIO_highbit64() : 757 * gives position of highest bit. 758 * note : only works for v > 0 ! 759 */ 760 static unsigned FIO_highbit64(unsigned long long v) 761 { 762 unsigned count = 0; 763 assert(v != 0); 764 v >>= 1; 765 while (v) { v >>= 1; count++; } 766 return count; 767 } 768 769 static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, 770 unsigned long long const dictSize, 771 unsigned long long const maxSrcFileSize) 772 { 773 unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); 774 assert(maxSize != UTIL_FILESIZE_UNKNOWN); 775 if (maxSize > UINT_MAX) 776 EXM_THROW(42, "Can't handle files larger than %u GB\n", UINT_MAX/(1 GB) + 1); 777 FIO_setMemLimit(prefs, (unsigned)maxSize); 778 } 779 780 #ifndef ZSTD_NOCOMPRESS 781 782 /* ********************************************************************** 783 * Compression 784 ************************************************************************/ 785 typedef struct { 786 FILE* srcFile; 787 FILE* dstFile; 788 void* srcBuffer; 789 size_t srcBufferSize; 790 void* dstBuffer; 791 size_t dstBufferSize; 792 void* dictBuffer; 793 size_t dictBufferSize; 794 const char* dictFileName; 795 ZSTD_CStream* cctx; 796 } cRess_t; 797 798 static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs, 799 ZSTD_compressionParameters* comprParams, 800 unsigned long long const dictSize, 801 unsigned long long const maxSrcFileSize, 802 int cLevel) 803 { 804 unsigned const fileWindowLog = FIO_highbit64(maxSrcFileSize) + 1; 805 ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize); 806 FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize); 807 if (fileWindowLog > ZSTD_WINDOWLOG_MAX) 808 DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n"); 809 comprParams->windowLog = MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog); 810 if (fileWindowLog > ZSTD_cycleLog(cParams.hashLog, cParams.strategy)) { 811 if (!prefs->ldmFlag) 812 DISPLAYLEVEL(1, "long mode automaticaly triggered\n"); 813 FIO_setLdmFlag(prefs, 1); 814 } 815 if (cParams.strategy >= ZSTD_btopt) { 816 DISPLAYLEVEL(1, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n"); 817 DISPLAYLEVEL(1, "- Use --single-thread mode in the zstd cli\n"); 818 DISPLAYLEVEL(1, "- Set a larger targetLength (eg. --zstd=targetLength=4096)\n"); 819 DISPLAYLEVEL(1, "- Set a larger chainLog (eg. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); 820 DISPLAYLEVEL(1, "Also consdier playing around with searchLog and hashLog\n"); 821 } 822 } 823 824 static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, 825 const char* dictFileName, unsigned long long const maxSrcFileSize, 826 int cLevel, ZSTD_compressionParameters comprParams) { 827 cRess_t ress; 828 memset(&ress, 0, sizeof(ress)); 829 830 DISPLAYLEVEL(6, "FIO_createCResources \n"); 831 ress.cctx = ZSTD_createCCtx(); 832 if (ress.cctx == NULL) 833 EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx", 834 strerror(errno)); 835 ress.srcBufferSize = ZSTD_CStreamInSize(); 836 ress.srcBuffer = malloc(ress.srcBufferSize); 837 ress.dstBufferSize = ZSTD_CStreamOutSize(); 838 839 /* need to update memLimit before calling createDictBuffer 840 * because of memLimit check inside it */ 841 if (prefs->patchFromMode) 842 FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), maxSrcFileSize, cLevel); 843 ress.dstBuffer = malloc(ress.dstBufferSize); 844 ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ 845 if (!ress.srcBuffer || !ress.dstBuffer) 846 EXM_THROW(31, "allocation error : not enough memory"); 847 848 /* Advanced parameters, including dictionary */ 849 if (dictFileName && (ress.dictBuffer==NULL)) 850 EXM_THROW(32, "allocation error : can't create dictBuffer"); 851 ress.dictFileName = dictFileName; 852 853 if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) 854 comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; 855 856 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */ 857 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) ); 858 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) ); 859 /* compression level */ 860 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); 861 /* max compressed block size */ 862 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) ); 863 /* source size hint */ 864 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) ); 865 /* long distance matching */ 866 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) ); 867 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) ); 868 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) ); 869 if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { 870 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) ); 871 } 872 if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { 873 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); 874 } 875 /* compression parameters */ 876 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) ); 877 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) ); 878 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) ); 879 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) ); 880 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) ); 881 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); 882 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); 883 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); 884 /* multi-threading */ 885 #ifdef ZSTD_MULTITHREAD 886 DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); 887 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); 888 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); 889 if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { 890 DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); 891 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); 892 } 893 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); 894 #endif 895 /* dictionary */ 896 if (prefs->patchFromMode) { 897 CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); 898 } else { 899 CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); 900 } 901 902 return ress; 903 } 904 905 static void FIO_freeCResources(cRess_t ress) 906 { 907 free(ress.srcBuffer); 908 free(ress.dstBuffer); 909 free(ress.dictBuffer); 910 ZSTD_freeCStream(ress.cctx); /* never fails */ 911 } 912 913 914 #ifdef ZSTD_GZCOMPRESS 915 static unsigned long long 916 FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */ 917 const char* srcFileName, U64 const srcFileSize, 918 int compressionLevel, U64* readsize) 919 { 920 unsigned long long inFileSize = 0, outFileSize = 0; 921 z_stream strm; 922 923 if (compressionLevel > Z_BEST_COMPRESSION) 924 compressionLevel = Z_BEST_COMPRESSION; 925 926 strm.zalloc = Z_NULL; 927 strm.zfree = Z_NULL; 928 strm.opaque = Z_NULL; 929 930 { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 931 15 /* maxWindowLogSize */ + 16 /* gzip only */, 932 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 933 if (ret != Z_OK) { 934 EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); 935 } } 936 937 strm.next_in = 0; 938 strm.avail_in = 0; 939 strm.next_out = (Bytef*)ress->dstBuffer; 940 strm.avail_out = (uInt)ress->dstBufferSize; 941 942 while (1) { 943 int ret; 944 if (strm.avail_in == 0) { 945 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 946 if (inSize == 0) break; 947 inFileSize += inSize; 948 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 949 strm.avail_in = (uInt)inSize; 950 } 951 ret = deflate(&strm, Z_NO_FLUSH); 952 if (ret != Z_OK) 953 EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); 954 { size_t const cSize = ress->dstBufferSize - strm.avail_out; 955 if (cSize) { 956 if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) 957 EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno)); 958 outFileSize += cSize; 959 strm.next_out = (Bytef*)ress->dstBuffer; 960 strm.avail_out = (uInt)ress->dstBufferSize; 961 } } 962 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { 963 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", 964 (unsigned)(inFileSize>>20), 965 (double)outFileSize/inFileSize*100) 966 } else { 967 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ", 968 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 969 (double)outFileSize/inFileSize*100); 970 } } 971 972 while (1) { 973 int const ret = deflate(&strm, Z_FINISH); 974 { size_t const cSize = ress->dstBufferSize - strm.avail_out; 975 if (cSize) { 976 if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) 977 EXM_THROW(75, "Write error : %s ", strerror(errno)); 978 outFileSize += cSize; 979 strm.next_out = (Bytef*)ress->dstBuffer; 980 strm.avail_out = (uInt)ress->dstBufferSize; 981 } } 982 if (ret == Z_STREAM_END) break; 983 if (ret != Z_BUF_ERROR) 984 EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); 985 } 986 987 { int const ret = deflateEnd(&strm); 988 if (ret != Z_OK) { 989 EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); 990 } } 991 *readsize = inFileSize; 992 return outFileSize; 993 } 994 #endif 995 996 997 #ifdef ZSTD_LZMACOMPRESS 998 static unsigned long long 999 FIO_compressLzmaFrame(cRess_t* ress, 1000 const char* srcFileName, U64 const srcFileSize, 1001 int compressionLevel, U64* readsize, int plain_lzma) 1002 { 1003 unsigned long long inFileSize = 0, outFileSize = 0; 1004 lzma_stream strm = LZMA_STREAM_INIT; 1005 lzma_action action = LZMA_RUN; 1006 lzma_ret ret; 1007 1008 if (compressionLevel < 0) compressionLevel = 0; 1009 if (compressionLevel > 9) compressionLevel = 9; 1010 1011 if (plain_lzma) { 1012 lzma_options_lzma opt_lzma; 1013 if (lzma_lzma_preset(&opt_lzma, compressionLevel)) 1014 EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName); 1015 ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ 1016 if (ret != LZMA_OK) 1017 EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); 1018 } else { 1019 ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ 1020 if (ret != LZMA_OK) 1021 EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); 1022 } 1023 1024 strm.next_in = 0; 1025 strm.avail_in = 0; 1026 strm.next_out = (BYTE*)ress->dstBuffer; 1027 strm.avail_out = ress->dstBufferSize; 1028 1029 while (1) { 1030 if (strm.avail_in == 0) { 1031 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 1032 if (inSize == 0) action = LZMA_FINISH; 1033 inFileSize += inSize; 1034 strm.next_in = (BYTE const*)ress->srcBuffer; 1035 strm.avail_in = inSize; 1036 } 1037 1038 ret = lzma_code(&strm, action); 1039 1040 if (ret != LZMA_OK && ret != LZMA_STREAM_END) 1041 EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); 1042 { size_t const compBytes = ress->dstBufferSize - strm.avail_out; 1043 if (compBytes) { 1044 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) 1045 EXM_THROW(85, "Write error : %s", strerror(errno)); 1046 outFileSize += compBytes; 1047 strm.next_out = (BYTE*)ress->dstBuffer; 1048 strm.avail_out = ress->dstBufferSize; 1049 } } 1050 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 1051 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 1052 (unsigned)(inFileSize>>20), 1053 (double)outFileSize/inFileSize*100) 1054 else 1055 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 1056 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 1057 (double)outFileSize/inFileSize*100); 1058 if (ret == LZMA_STREAM_END) break; 1059 } 1060 1061 lzma_end(&strm); 1062 *readsize = inFileSize; 1063 1064 return outFileSize; 1065 } 1066 #endif 1067 1068 #ifdef ZSTD_LZ4COMPRESS 1069 1070 #if LZ4_VERSION_NUMBER <= 10600 1071 #define LZ4F_blockLinked blockLinked 1072 #define LZ4F_max64KB max64KB 1073 #endif 1074 1075 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 1076 1077 static unsigned long long 1078 FIO_compressLz4Frame(cRess_t* ress, 1079 const char* srcFileName, U64 const srcFileSize, 1080 int compressionLevel, int checksumFlag, 1081 U64* readsize) 1082 { 1083 const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); 1084 unsigned long long inFileSize = 0, outFileSize = 0; 1085 1086 LZ4F_preferences_t prefs; 1087 LZ4F_compressionContext_t ctx; 1088 1089 LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 1090 if (LZ4F_isError(errorCode)) 1091 EXM_THROW(31, "zstd: failed to create lz4 compression context"); 1092 1093 memset(&prefs, 0, sizeof(prefs)); 1094 1095 assert(blockSize <= ress->srcBufferSize); 1096 1097 prefs.autoFlush = 1; 1098 prefs.compressionLevel = compressionLevel; 1099 prefs.frameInfo.blockMode = LZ4F_blockLinked; 1100 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 1101 prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag; 1102 #if LZ4_VERSION_NUMBER >= 10600 1103 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; 1104 #endif 1105 assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); 1106 1107 { 1108 size_t readSize; 1109 size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); 1110 if (LZ4F_isError(headerSize)) 1111 EXM_THROW(33, "File header generation failed : %s", 1112 LZ4F_getErrorName(headerSize)); 1113 if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) 1114 EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno)); 1115 outFileSize += headerSize; 1116 1117 /* Read first block */ 1118 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 1119 inFileSize += readSize; 1120 1121 /* Main Loop */ 1122 while (readSize>0) { 1123 size_t const outSize = LZ4F_compressUpdate(ctx, 1124 ress->dstBuffer, ress->dstBufferSize, 1125 ress->srcBuffer, readSize, NULL); 1126 if (LZ4F_isError(outSize)) 1127 EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", 1128 srcFileName, LZ4F_getErrorName(outSize)); 1129 outFileSize += outSize; 1130 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { 1131 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 1132 (unsigned)(inFileSize>>20), 1133 (double)outFileSize/inFileSize*100) 1134 } else { 1135 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 1136 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 1137 (double)outFileSize/inFileSize*100); 1138 } 1139 1140 /* Write Block */ 1141 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); 1142 if (sizeCheck != outSize) 1143 EXM_THROW(36, "Write error : %s", strerror(errno)); 1144 } 1145 1146 /* Read next block */ 1147 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 1148 inFileSize += readSize; 1149 } 1150 if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); 1151 1152 /* End of Stream mark */ 1153 headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); 1154 if (LZ4F_isError(headerSize)) 1155 EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", 1156 srcFileName, LZ4F_getErrorName(headerSize)); 1157 1158 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); 1159 if (sizeCheck != headerSize) 1160 EXM_THROW(39, "Write error : %s (cannot write end of stream)", 1161 strerror(errno)); 1162 } 1163 outFileSize += headerSize; 1164 } 1165 1166 *readsize = inFileSize; 1167 LZ4F_freeCompressionContext(ctx); 1168 1169 return outFileSize; 1170 } 1171 #endif 1172 1173 1174 static unsigned long long 1175 FIO_compressZstdFrame(FIO_prefs_t* const prefs, 1176 const cRess_t* ressPtr, 1177 const char* srcFileName, U64 fileSize, 1178 int compressionLevel, U64* readsize) 1179 { 1180 cRess_t const ress = *ressPtr; 1181 FILE* const srcFile = ress.srcFile; 1182 FILE* const dstFile = ress.dstFile; 1183 U64 compressedfilesize = 0; 1184 ZSTD_EndDirective directive = ZSTD_e_continue; 1185 1186 /* stats */ 1187 ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 }; 1188 ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 }; 1189 typedef enum { noChange, slower, faster } speedChange_e; 1190 speedChange_e speedChange = noChange; 1191 unsigned flushWaiting = 0; 1192 unsigned inputPresented = 0; 1193 unsigned inputBlocked = 0; 1194 unsigned lastJobID = 0; 1195 1196 DISPLAYLEVEL(6, "compression using zstd format \n"); 1197 1198 /* init */ 1199 if (fileSize != UTIL_FILESIZE_UNKNOWN) { 1200 CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); 1201 } else if (prefs->streamSrcSize > 0) { 1202 /* unknown source size; use the declared stream size */ 1203 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); 1204 } 1205 (void)srcFileName; 1206 1207 /* Main compression loop */ 1208 do { 1209 size_t stillToFlush; 1210 /* Fill input Buffer */ 1211 size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); 1212 ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; 1213 DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); 1214 *readsize += inSize; 1215 1216 if ((inSize == 0) || (*readsize == fileSize)) 1217 directive = ZSTD_e_end; 1218 1219 stillToFlush = 1; 1220 while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ 1221 || (directive == ZSTD_e_end && stillToFlush != 0) ) { 1222 1223 size_t const oldIPos = inBuff.pos; 1224 ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; 1225 size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); 1226 CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); 1227 1228 /* count stats */ 1229 inputPresented++; 1230 if (oldIPos == inBuff.pos) inputBlocked++; /* input buffer is full and can't take any more : input speed is faster than consumption rate */ 1231 if (!toFlushNow) flushWaiting = 1; 1232 1233 /* Write compressed stream */ 1234 DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", 1235 (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); 1236 if (outBuff.pos) { 1237 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); 1238 if (sizeCheck != outBuff.pos) 1239 EXM_THROW(25, "Write error : %s (cannot write compressed block)", 1240 strerror(errno)); 1241 compressedfilesize += outBuff.pos; 1242 } 1243 1244 /* display notification; and adapt compression level */ 1245 if (READY_FOR_UPDATE()) { 1246 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); 1247 double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; 1248 1249 /* display progress notifications */ 1250 if (g_display_prefs.displayLevel >= 3) { 1251 DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", 1252 compressionLevel, 1253 (unsigned)((zfp.ingested - zfp.consumed) >> 20), 1254 (unsigned)(zfp.consumed >> 20), 1255 (unsigned)(zfp.produced >> 20), 1256 cShare ); 1257 } else { /* summarized notifications if == 2; */ 1258 DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20)); 1259 if (fileSize != UTIL_FILESIZE_UNKNOWN) 1260 DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20)); 1261 DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare); 1262 DELAY_NEXT_UPDATE(); 1263 } 1264 1265 /* adaptive mode : statistics measurement and speed correction */ 1266 if (prefs->adaptiveMode) { 1267 1268 /* check output speed */ 1269 if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ 1270 1271 unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; 1272 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; 1273 assert(zfp.produced >= previous_zfp_update.produced); 1274 assert(prefs->nbWorkers >= 1); 1275 1276 /* test if compression is blocked 1277 * either because output is slow and all buffers are full 1278 * or because input is slow and no job can start while waiting for at least one buffer to be filled. 1279 * note : exclude starting part, since currentJobID > 1 */ 1280 if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/ 1281 && (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */ 1282 ) { 1283 DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n") 1284 speedChange = slower; 1285 } 1286 1287 previous_zfp_update = zfp; 1288 1289 if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */ 1290 && (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */ 1291 ) { 1292 DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed); 1293 speedChange = slower; 1294 } 1295 flushWaiting = 0; 1296 } 1297 1298 /* course correct only if there is at least one new job completed */ 1299 if (zfp.currentJobID > lastJobID) { 1300 DISPLAYLEVEL(6, "compression level adaptation check \n") 1301 1302 /* check input speed */ 1303 if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */ 1304 if (inputBlocked <= 0) { 1305 DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); 1306 speedChange = slower; 1307 } else if (speedChange == noChange) { 1308 unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested; 1309 unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed; 1310 unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced; 1311 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed; 1312 previous_zfp_correction = zfp; 1313 assert(inputPresented > 0); 1314 DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", 1315 inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, 1316 (unsigned)newlyIngested, (unsigned)newlyConsumed, 1317 (unsigned)newlyFlushed, (unsigned)newlyProduced); 1318 if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ 1319 && (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */ 1320 && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */ 1321 ) { 1322 DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", 1323 newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); 1324 speedChange = faster; 1325 } 1326 } 1327 inputBlocked = 0; 1328 inputPresented = 0; 1329 } 1330 1331 if (speedChange == slower) { 1332 DISPLAYLEVEL(6, "slower speed , higher compression \n") 1333 compressionLevel ++; 1334 if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); 1335 if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel; 1336 compressionLevel += (compressionLevel == 0); /* skip 0 */ 1337 ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); 1338 } 1339 if (speedChange == faster) { 1340 DISPLAYLEVEL(6, "faster speed , lighter compression \n") 1341 compressionLevel --; 1342 if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel; 1343 compressionLevel -= (compressionLevel == 0); /* skip 0 */ 1344 ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); 1345 } 1346 speedChange = noChange; 1347 1348 lastJobID = zfp.currentJobID; 1349 } /* if (zfp.currentJobID > lastJobID) */ 1350 } /* if (g_adaptiveMode) */ 1351 } /* if (READY_FOR_UPDATE()) */ 1352 } /* while ((inBuff.pos != inBuff.size) */ 1353 } while (directive != ZSTD_e_end); 1354 1355 if (ferror(srcFile)) { 1356 EXM_THROW(26, "Read error : I/O error"); 1357 } 1358 if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { 1359 EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", 1360 (unsigned long long)*readsize, (unsigned long long)fileSize); 1361 } 1362 1363 return compressedfilesize; 1364 } 1365 1366 /*! FIO_compressFilename_internal() : 1367 * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. 1368 * @return : 0 : compression completed correctly, 1369 * 1 : missing or pb opening srcFileName 1370 */ 1371 static int 1372 FIO_compressFilename_internal(FIO_prefs_t* const prefs, 1373 cRess_t ress, 1374 const char* dstFileName, const char* srcFileName, 1375 int compressionLevel) 1376 { 1377 UTIL_time_t const timeStart = UTIL_getTime(); 1378 clock_t const cpuStart = clock(); 1379 U64 readsize = 0; 1380 U64 compressedfilesize = 0; 1381 U64 const fileSize = UTIL_getFileSize(srcFileName); 1382 DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize); 1383 1384 /* compression format selection */ 1385 switch (prefs->compressionType) { 1386 default: 1387 case FIO_zstdCompression: 1388 compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); 1389 break; 1390 1391 case FIO_gzipCompression: 1392 #ifdef ZSTD_GZCOMPRESS 1393 compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 1394 #else 1395 (void)compressionLevel; 1396 EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", 1397 srcFileName); 1398 #endif 1399 break; 1400 1401 case FIO_xzCompression: 1402 case FIO_lzmaCompression: 1403 #ifdef ZSTD_LZMACOMPRESS 1404 compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression); 1405 #else 1406 (void)compressionLevel; 1407 EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", 1408 srcFileName); 1409 #endif 1410 break; 1411 1412 case FIO_lz4Compression: 1413 #ifdef ZSTD_LZ4COMPRESS 1414 compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize); 1415 #else 1416 (void)compressionLevel; 1417 EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", 1418 srcFileName); 1419 #endif 1420 break; 1421 } 1422 1423 /* Status */ 1424 DISPLAYLEVEL(2, "\r%79s\r", ""); 1425 if (readsize == 0) { 1426 DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", 1427 srcFileName, 1428 (unsigned long long)readsize, (unsigned long long) compressedfilesize, 1429 dstFileName); 1430 } else { 1431 DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", 1432 srcFileName, 1433 (double)compressedfilesize / readsize * 100, 1434 (unsigned long long)readsize, (unsigned long long) compressedfilesize, 1435 dstFileName); 1436 } 1437 1438 /* Elapsed Time and CPU Load */ 1439 { clock_t const cpuEnd = clock(); 1440 double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC; 1441 U64 const timeLength_ns = UTIL_clockSpanNano(timeStart); 1442 double const timeLength_s = (double)timeLength_ns / 1000000000; 1443 double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100; 1444 DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n", 1445 srcFileName, timeLength_s, cpuLoad_pct); 1446 } 1447 return 0; 1448 } 1449 1450 1451 /*! FIO_compressFilename_dstFile() : 1452 * open dstFileName, or pass-through if ress.dstFile != NULL, 1453 * then start compression with FIO_compressFilename_internal(). 1454 * Manages source removal (--rm) and file permissions transfer. 1455 * note : ress.srcFile must be != NULL, 1456 * so reach this function through FIO_compressFilename_srcFile(). 1457 * @return : 0 : compression completed correctly, 1458 * 1 : pb 1459 */ 1460 static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, 1461 cRess_t ress, 1462 const char* dstFileName, 1463 const char* srcFileName, 1464 int compressionLevel) 1465 { 1466 int closeDstFile = 0; 1467 int result; 1468 stat_t statbuf; 1469 int transfer_permissions = 0; 1470 assert(ress.srcFile != NULL); 1471 if (ress.dstFile == NULL) { 1472 closeDstFile = 1; 1473 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); 1474 ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); 1475 if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ 1476 /* Must only be added after FIO_openDstFile() succeeds. 1477 * Otherwise we may delete the destination file if it already exists, 1478 * and the user presses Ctrl-C when asked if they wish to overwrite. 1479 */ 1480 addHandler(dstFileName); 1481 1482 if ( strcmp (srcFileName, stdinmark) 1483 && UTIL_getFileStat(srcFileName, &statbuf)) 1484 transfer_permissions = 1; 1485 } 1486 1487 result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel); 1488 1489 if (closeDstFile) { 1490 FILE* const dstFile = ress.dstFile; 1491 ress.dstFile = NULL; 1492 1493 clearHandler(); 1494 1495 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); 1496 if (fclose(dstFile)) { /* error closing dstFile */ 1497 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 1498 result=1; 1499 } 1500 if ( (result != 0) /* operation failure */ 1501 && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null */ 1502 && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ 1503 ) { 1504 FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ 1505 } else if ( strcmp(dstFileName, stdoutmark) 1506 && strcmp(dstFileName, nulmark) 1507 && transfer_permissions) { 1508 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transfering permissions into dst: %s \n", dstFileName); 1509 UTIL_setFileStat(dstFileName, &statbuf); 1510 } else { 1511 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: do not transfer permissions into dst: %s \n", dstFileName); 1512 } 1513 } 1514 1515 return result; 1516 } 1517 1518 /* List used to compare file extensions (used with --exclude-compressed flag) 1519 * Different from the suffixList and should only apply to ZSTD compress operationResult 1520 */ 1521 static const char *compressedFileExtensions[] = { 1522 ZSTD_EXTENSION, 1523 TZSTD_EXTENSION, 1524 GZ_EXTENSION, 1525 TGZ_EXTENSION, 1526 LZMA_EXTENSION, 1527 XZ_EXTENSION, 1528 TXZ_EXTENSION, 1529 LZ4_EXTENSION, 1530 TLZ4_EXTENSION, 1531 NULL 1532 }; 1533 1534 /*! FIO_compressFilename_srcFile() : 1535 * @return : 0 : compression completed correctly, 1536 * 1 : missing or pb opening srcFileName 1537 */ 1538 static int 1539 FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, 1540 cRess_t ress, 1541 const char* dstFileName, 1542 const char* srcFileName, 1543 int compressionLevel) 1544 { 1545 int result; 1546 DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName); 1547 1548 /* ensure src is not a directory */ 1549 if (UTIL_isDirectory(srcFileName)) { 1550 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 1551 return 1; 1552 } 1553 1554 /* ensure src is not the same as dict (if present) */ 1555 if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) { 1556 DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); 1557 return 1; 1558 } 1559 1560 /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used 1561 * YES => ZSTD will skip compression of the file and will return 0. 1562 * NO => ZSTD will resume with compress operation. 1563 */ 1564 if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) { 1565 DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName); 1566 return 0; 1567 } 1568 1569 ress.srcFile = FIO_openSrcFile(srcFileName); 1570 if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ 1571 1572 result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel); 1573 1574 fclose(ress.srcFile); 1575 ress.srcFile = NULL; 1576 if ( prefs->removeSrcFile /* --rm */ 1577 && result == 0 /* success */ 1578 && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ 1579 ) { 1580 /* We must clear the handler, since after this point calling it would 1581 * delete both the source and destination files. 1582 */ 1583 clearHandler(); 1584 if (FIO_remove(srcFileName)) 1585 EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); 1586 } 1587 return result; 1588 } 1589 1590 int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, 1591 const char* srcFileName, const char* dictFileName, 1592 int compressionLevel, ZSTD_compressionParameters comprParams) 1593 { 1594 cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); 1595 int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); 1596 1597 1598 FIO_freeCResources(ress); 1599 return result; 1600 } 1601 1602 /* FIO_determineCompressedName() : 1603 * create a destination filename for compressed srcFileName. 1604 * @return a pointer to it. 1605 * This function never returns an error (it may abort() in case of pb) 1606 */ 1607 static const char* 1608 FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix) 1609 { 1610 static size_t dfnbCapacity = 0; 1611 static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ 1612 char* outDirFilename = NULL; 1613 size_t sfnSize = strlen(srcFileName); 1614 size_t const srcSuffixLen = strlen(suffix); 1615 if (outDirName) { 1616 outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen); 1617 sfnSize = strlen(outDirFilename); 1618 assert(outDirFilename != NULL); 1619 } 1620 1621 if (dfnbCapacity <= sfnSize+srcSuffixLen+1) { 1622 /* resize buffer for dstName */ 1623 free(dstFileNameBuffer); 1624 dfnbCapacity = sfnSize + srcSuffixLen + 30; 1625 dstFileNameBuffer = (char*)malloc(dfnbCapacity); 1626 if (!dstFileNameBuffer) { 1627 EXM_THROW(30, "zstd: %s", strerror(errno)); 1628 } 1629 } 1630 assert(dstFileNameBuffer != NULL); 1631 1632 if (outDirFilename) { 1633 memcpy(dstFileNameBuffer, outDirFilename, sfnSize); 1634 free(outDirFilename); 1635 } else { 1636 memcpy(dstFileNameBuffer, srcFileName, sfnSize); 1637 } 1638 memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */); 1639 return dstFileNameBuffer; 1640 } 1641 1642 static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsigned nbFiles) 1643 { 1644 size_t i; 1645 unsigned long long fileSize, maxFileSize = 0; 1646 for (i = 0; i < nbFiles; i++) { 1647 fileSize = UTIL_getFileSize(inFileNames[i]); 1648 maxFileSize = fileSize > maxFileSize ? fileSize : maxFileSize; 1649 } 1650 return maxFileSize; 1651 } 1652 1653 /* FIO_compressMultipleFilenames() : 1654 * compress nbFiles files 1655 * into either one destination (outFileName), 1656 * or into one file each (outFileName == NULL, but suffix != NULL), 1657 * or into a destination folder (specified with -O) 1658 */ 1659 int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, 1660 const char** inFileNamesTable, unsigned nbFiles, 1661 const char* outDirName, 1662 const char* outFileName, const char* suffix, 1663 const char* dictFileName, int compressionLevel, 1664 ZSTD_compressionParameters comprParams) 1665 { 1666 int error = 0; 1667 cRess_t ress = FIO_createCResources(prefs, dictFileName, 1668 FIO_getLargestFileSize(inFileNamesTable, nbFiles), 1669 compressionLevel, comprParams); 1670 1671 /* init */ 1672 assert(outFileName != NULL || suffix != NULL); 1673 if (outFileName != NULL) { /* output into a single destination (stdout typically) */ 1674 ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); 1675 if (ress.dstFile == NULL) { /* could not open outFileName */ 1676 error = 1; 1677 } else { 1678 unsigned u; 1679 for (u=0; u<nbFiles; u++) 1680 error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); 1681 if (fclose(ress.dstFile)) 1682 EXM_THROW(29, "Write error (%s) : cannot properly close %s", 1683 strerror(errno), outFileName); 1684 ress.dstFile = NULL; 1685 } 1686 } else { 1687 unsigned u; 1688 for (u=0; u<nbFiles; u++) { 1689 const char* const srcFileName = inFileNamesTable[u]; 1690 const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ 1691 error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); 1692 } 1693 if (outDirName) 1694 FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); 1695 } 1696 1697 FIO_freeCResources(ress); 1698 return error; 1699 } 1700 1701 #endif /* #ifndef ZSTD_NOCOMPRESS */ 1702 1703 1704 1705 #ifndef ZSTD_NODECOMPRESS 1706 1707 /* ************************************************************************** 1708 * Decompression 1709 ***************************************************************************/ 1710 typedef struct { 1711 void* srcBuffer; 1712 size_t srcBufferSize; 1713 size_t srcBufferLoaded; 1714 void* dstBuffer; 1715 size_t dstBufferSize; 1716 ZSTD_DStream* dctx; 1717 FILE* dstFile; 1718 } dRess_t; 1719 1720 static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) 1721 { 1722 dRess_t ress; 1723 memset(&ress, 0, sizeof(ress)); 1724 1725 if (prefs->patchFromMode) 1726 FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */); 1727 1728 /* Allocation */ 1729 ress.dctx = ZSTD_createDStream(); 1730 if (ress.dctx==NULL) 1731 EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); 1732 CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); 1733 ress.srcBufferSize = ZSTD_DStreamInSize(); 1734 ress.srcBuffer = malloc(ress.srcBufferSize); 1735 ress.dstBufferSize = ZSTD_DStreamOutSize(); 1736 ress.dstBuffer = malloc(ress.dstBufferSize); 1737 if (!ress.srcBuffer || !ress.dstBuffer) 1738 EXM_THROW(61, "Allocation error : not enough memory"); 1739 1740 /* dictionary */ 1741 { void* dictBuffer; 1742 size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs); 1743 CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); 1744 free(dictBuffer); 1745 } 1746 1747 return ress; 1748 } 1749 1750 static void FIO_freeDResources(dRess_t ress) 1751 { 1752 CHECK( ZSTD_freeDStream(ress.dctx) ); 1753 free(ress.srcBuffer); 1754 free(ress.dstBuffer); 1755 } 1756 1757 1758 /** FIO_fwriteSparse() : 1759 * @return : storedSkips, 1760 * argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ 1761 static unsigned 1762 FIO_fwriteSparse(FILE* file, 1763 const void* buffer, size_t bufferSize, 1764 const FIO_prefs_t* const prefs, 1765 unsigned storedSkips) 1766 { 1767 const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 1768 size_t bufferSizeT = bufferSize / sizeof(size_t); 1769 const size_t* const bufferTEnd = bufferT + bufferSizeT; 1770 const size_t* ptrT = bufferT; 1771 static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ 1772 1773 if (prefs->testMode) return 0; /* do not output anything in test mode */ 1774 1775 if (!prefs->sparseFileSupport) { /* normal write */ 1776 size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); 1777 if (sizeCheck != bufferSize) 1778 EXM_THROW(70, "Write error : cannot write decoded block : %s", 1779 strerror(errno)); 1780 return 0; 1781 } 1782 1783 /* avoid int overflow */ 1784 if (storedSkips > 1 GB) { 1785 if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) 1786 EXM_THROW(91, "1 GB skip error (sparse file support)"); 1787 storedSkips -= 1 GB; 1788 } 1789 1790 while (ptrT < bufferTEnd) { 1791 size_t nb0T; 1792 1793 /* adjust last segment if < 32 KB */ 1794 size_t seg0SizeT = segmentSizeT; 1795 if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 1796 bufferSizeT -= seg0SizeT; 1797 1798 /* count leading zeroes */ 1799 for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 1800 storedSkips += (unsigned)(nb0T * sizeof(size_t)); 1801 1802 if (nb0T != seg0SizeT) { /* not all 0s */ 1803 size_t const nbNon0ST = seg0SizeT - nb0T; 1804 /* skip leading zeros */ 1805 if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) 1806 EXM_THROW(92, "Sparse skip error ; try --no-sparse"); 1807 storedSkips = 0; 1808 /* write the rest */ 1809 if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) 1810 EXM_THROW(93, "Write error : cannot write decoded block : %s", 1811 strerror(errno)); 1812 } 1813 ptrT += seg0SizeT; 1814 } 1815 1816 { static size_t const maskT = sizeof(size_t)-1; 1817 if (bufferSize & maskT) { 1818 /* size not multiple of sizeof(size_t) : implies end of block */ 1819 const char* const restStart = (const char*)bufferTEnd; 1820 const char* restPtr = restStart; 1821 const char* const restEnd = (const char*)buffer + bufferSize; 1822 assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); 1823 for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 1824 storedSkips += (unsigned) (restPtr - restStart); 1825 if (restPtr != restEnd) { 1826 /* not all remaining bytes are 0 */ 1827 size_t const restSize = (size_t)(restEnd - restPtr); 1828 if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) 1829 EXM_THROW(92, "Sparse skip error ; try --no-sparse"); 1830 if (fwrite(restPtr, 1, restSize, file) != restSize) 1831 EXM_THROW(95, "Write error : cannot write end of decoded block : %s", 1832 strerror(errno)); 1833 storedSkips = 0; 1834 } } } 1835 1836 return storedSkips; 1837 } 1838 1839 static void 1840 FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) 1841 { 1842 if (prefs->testMode) assert(storedSkips == 0); 1843 if (storedSkips>0) { 1844 assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ 1845 (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ 1846 if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) 1847 EXM_THROW(69, "Final skip error (sparse file support)"); 1848 /* last zero must be explicitly written, 1849 * so that skipped ones get implicitly translated as zero by FS */ 1850 { const char lastZeroByte[1] = { 0 }; 1851 if (fwrite(lastZeroByte, 1, 1, file) != 1) 1852 EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); 1853 } } 1854 } 1855 1856 1857 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode 1858 @return : 0 (no error) */ 1859 static int FIO_passThrough(const FIO_prefs_t* const prefs, 1860 FILE* foutput, FILE* finput, 1861 void* buffer, size_t bufferSize, 1862 size_t alreadyLoaded) 1863 { 1864 size_t const blockSize = MIN(64 KB, bufferSize); 1865 size_t readFromInput; 1866 unsigned storedSkips = 0; 1867 1868 /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ 1869 { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); 1870 if (sizeCheck != alreadyLoaded) { 1871 DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno)); 1872 return 1; 1873 } } 1874 1875 do { 1876 readFromInput = fread(buffer, 1, blockSize, finput); 1877 storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, prefs, storedSkips); 1878 } while (readFromInput == blockSize); 1879 if (ferror(finput)) { 1880 DISPLAYLEVEL(1, "Pass-through read error : %s\n", strerror(errno)); 1881 return 1; 1882 } 1883 assert(feof(finput)); 1884 1885 FIO_fwriteSparseEnd(prefs, foutput, storedSkips); 1886 return 0; 1887 } 1888 1889 /* FIO_zstdErrorHelp() : 1890 * detailed error message when requested window size is too large */ 1891 static void 1892 FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, 1893 const dRess_t* ress, 1894 size_t err, const char* srcFileName) 1895 { 1896 ZSTD_frameHeader header; 1897 1898 /* Help message only for one specific error */ 1899 if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge) 1900 return; 1901 1902 /* Try to decode the frame header */ 1903 err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); 1904 if (err == 0) { 1905 unsigned long long const windowSize = header.windowSize; 1906 unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); 1907 assert(prefs->memLimit > 0); 1908 DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n", 1909 srcFileName, windowSize, prefs->memLimit); 1910 if (windowLog <= ZSTD_WINDOWLOG_MAX) { 1911 unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); 1912 assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ 1913 DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n", 1914 srcFileName, windowLog, windowMB); 1915 return; 1916 } } 1917 DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n", 1918 srcFileName, ZSTD_WINDOWLOG_MAX); 1919 } 1920 1921 /** FIO_decompressFrame() : 1922 * @return : size of decoded zstd frame, or an error code 1923 */ 1924 #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) 1925 static unsigned long long 1926 FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, 1927 const FIO_prefs_t* const prefs, 1928 const char* srcFileName, 1929 U64 alreadyDecoded) /* for multi-frames streams */ 1930 { 1931 U64 frameSize = 0; 1932 U32 storedSkips = 0; 1933 1934 /* display last 20 characters only */ 1935 { size_t const srcFileLength = strlen(srcFileName); 1936 if (srcFileLength>20) srcFileName += srcFileLength-20; 1937 } 1938 1939 ZSTD_resetDStream(ress->dctx); 1940 1941 /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ 1942 { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; 1943 if (ress->srcBufferLoaded < toDecode) { 1944 size_t const toRead = toDecode - ress->srcBufferLoaded; 1945 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1946 ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); 1947 } } 1948 1949 /* Main decompression Loop */ 1950 while (1) { 1951 ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; 1952 ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; 1953 size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); 1954 if (ZSTD_isError(readSizeHint)) { 1955 DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", 1956 srcFileName, ZSTD_getErrorName(readSizeHint)); 1957 FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); 1958 return FIO_ERROR_FRAME_DECODING; 1959 } 1960 1961 /* Write block */ 1962 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); 1963 frameSize += outBuff.pos; 1964 DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", 1965 srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); 1966 1967 if (inBuff.pos > 0) { 1968 memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); 1969 ress->srcBufferLoaded -= inBuff.pos; 1970 } 1971 1972 if (readSizeHint == 0) break; /* end of frame */ 1973 1974 /* Fill input buffer */ 1975 { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ 1976 if (ress->srcBufferLoaded < toDecode) { 1977 size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ 1978 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1979 size_t const readSize = fread(startPosition, 1, toRead, finput); 1980 if (readSize==0) { 1981 DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", 1982 srcFileName); 1983 return FIO_ERROR_FRAME_DECODING; 1984 } 1985 ress->srcBufferLoaded += readSize; 1986 } } } 1987 1988 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 1989 1990 return frameSize; 1991 } 1992 1993 1994 #ifdef ZSTD_GZDECOMPRESS 1995 static unsigned long long 1996 FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, 1997 const FIO_prefs_t* const prefs, 1998 const char* srcFileName) 1999 { 2000 unsigned long long outFileSize = 0; 2001 z_stream strm; 2002 int flush = Z_NO_FLUSH; 2003 int decodingError = 0; 2004 unsigned storedSkips = 0; 2005 2006 strm.zalloc = Z_NULL; 2007 strm.zfree = Z_NULL; 2008 strm.opaque = Z_NULL; 2009 strm.next_in = 0; 2010 strm.avail_in = 0; 2011 /* see http://www.zlib.net/manual.html */ 2012 if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) 2013 return FIO_ERROR_FRAME_DECODING; 2014 2015 strm.next_out = (Bytef*)ress->dstBuffer; 2016 strm.avail_out = (uInt)ress->dstBufferSize; 2017 strm.avail_in = (uInt)ress->srcBufferLoaded; 2018 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 2019 2020 for ( ; ; ) { 2021 int ret; 2022 if (strm.avail_in == 0) { 2023 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 2024 if (ress->srcBufferLoaded == 0) flush = Z_FINISH; 2025 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 2026 strm.avail_in = (uInt)ress->srcBufferLoaded; 2027 } 2028 ret = inflate(&strm, flush); 2029 if (ret == Z_BUF_ERROR) { 2030 DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); 2031 decodingError = 1; break; 2032 } 2033 if (ret != Z_OK && ret != Z_STREAM_END) { 2034 DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); 2035 decodingError = 1; break; 2036 } 2037 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 2038 if (decompBytes) { 2039 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); 2040 outFileSize += decompBytes; 2041 strm.next_out = (Bytef*)ress->dstBuffer; 2042 strm.avail_out = (uInt)ress->dstBufferSize; 2043 } 2044 } 2045 if (ret == Z_STREAM_END) break; 2046 } 2047 2048 if (strm.avail_in > 0) 2049 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 2050 ress->srcBufferLoaded = strm.avail_in; 2051 if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ 2052 && (decodingError==0) ) { 2053 DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); 2054 decodingError = 1; 2055 } 2056 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2057 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 2058 } 2059 #endif 2060 2061 2062 #ifdef ZSTD_LZMADECOMPRESS 2063 static unsigned long long 2064 FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, 2065 const FIO_prefs_t* const prefs, 2066 const char* srcFileName, int plain_lzma) 2067 { 2068 unsigned long long outFileSize = 0; 2069 lzma_stream strm = LZMA_STREAM_INIT; 2070 lzma_action action = LZMA_RUN; 2071 lzma_ret initRet; 2072 int decodingError = 0; 2073 unsigned storedSkips = 0; 2074 2075 strm.next_in = 0; 2076 strm.avail_in = 0; 2077 if (plain_lzma) { 2078 initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ 2079 } else { 2080 initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ 2081 } 2082 2083 if (initRet != LZMA_OK) { 2084 DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", 2085 plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", 2086 srcFileName, initRet); 2087 return FIO_ERROR_FRAME_DECODING; 2088 } 2089 2090 strm.next_out = (BYTE*)ress->dstBuffer; 2091 strm.avail_out = ress->dstBufferSize; 2092 strm.next_in = (BYTE const*)ress->srcBuffer; 2093 strm.avail_in = ress->srcBufferLoaded; 2094 2095 for ( ; ; ) { 2096 lzma_ret ret; 2097 if (strm.avail_in == 0) { 2098 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 2099 if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; 2100 strm.next_in = (BYTE const*)ress->srcBuffer; 2101 strm.avail_in = ress->srcBufferLoaded; 2102 } 2103 ret = lzma_code(&strm, action); 2104 2105 if (ret == LZMA_BUF_ERROR) { 2106 DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); 2107 decodingError = 1; break; 2108 } 2109 if (ret != LZMA_OK && ret != LZMA_STREAM_END) { 2110 DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", 2111 srcFileName, ret); 2112 decodingError = 1; break; 2113 } 2114 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 2115 if (decompBytes) { 2116 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); 2117 outFileSize += decompBytes; 2118 strm.next_out = (BYTE*)ress->dstBuffer; 2119 strm.avail_out = ress->dstBufferSize; 2120 } } 2121 if (ret == LZMA_STREAM_END) break; 2122 } 2123 2124 if (strm.avail_in > 0) 2125 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 2126 ress->srcBufferLoaded = strm.avail_in; 2127 lzma_end(&strm); 2128 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2129 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 2130 } 2131 #endif 2132 2133 #ifdef ZSTD_LZ4DECOMPRESS 2134 static unsigned long long 2135 FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, 2136 const FIO_prefs_t* const prefs, 2137 const char* srcFileName) 2138 { 2139 unsigned long long filesize = 0; 2140 LZ4F_errorCode_t nextToLoad; 2141 LZ4F_decompressionContext_t dCtx; 2142 LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 2143 int decodingError = 0; 2144 unsigned storedSkips = 0; 2145 2146 if (LZ4F_isError(errorCode)) { 2147 DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); 2148 return FIO_ERROR_FRAME_DECODING; 2149 } 2150 2151 /* Init feed with magic number (already consumed from FILE* sFile) */ 2152 { size_t inSize = 4; 2153 size_t outSize= 0; 2154 MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); 2155 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); 2156 if (LZ4F_isError(nextToLoad)) { 2157 DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", 2158 srcFileName, LZ4F_getErrorName(nextToLoad)); 2159 LZ4F_freeDecompressionContext(dCtx); 2160 return FIO_ERROR_FRAME_DECODING; 2161 } } 2162 2163 /* Main Loop */ 2164 for (;nextToLoad;) { 2165 size_t readSize; 2166 size_t pos = 0; 2167 size_t decodedBytes = ress->dstBufferSize; 2168 2169 /* Read input */ 2170 if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; 2171 readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); 2172 if (!readSize) break; /* reached end of file or stream */ 2173 2174 while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ 2175 /* Decode Input (at least partially) */ 2176 size_t remaining = readSize - pos; 2177 decodedBytes = ress->dstBufferSize; 2178 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); 2179 if (LZ4F_isError(nextToLoad)) { 2180 DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", 2181 srcFileName, LZ4F_getErrorName(nextToLoad)); 2182 decodingError = 1; nextToLoad = 0; break; 2183 } 2184 pos += remaining; 2185 2186 /* Write Block */ 2187 if (decodedBytes) { 2188 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); 2189 filesize += decodedBytes; 2190 DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); 2191 } 2192 2193 if (!nextToLoad) break; 2194 } 2195 } 2196 /* can be out because readSize == 0, which could be an fread() error */ 2197 if (ferror(srcFile)) { 2198 DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); 2199 decodingError=1; 2200 } 2201 2202 if (nextToLoad!=0) { 2203 DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); 2204 decodingError=1; 2205 } 2206 2207 LZ4F_freeDecompressionContext(dCtx); 2208 ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ 2209 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2210 2211 return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; 2212 } 2213 #endif 2214 2215 2216 2217 /** FIO_decompressFrames() : 2218 * Find and decode frames inside srcFile 2219 * srcFile presumed opened and valid 2220 * @return : 0 : OK 2221 * 1 : error 2222 */ 2223 static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, 2224 const FIO_prefs_t* const prefs, 2225 const char* dstFileName, const char* srcFileName) 2226 { 2227 unsigned readSomething = 0; 2228 unsigned long long filesize = 0; 2229 assert(srcFile != NULL); 2230 2231 /* for each frame */ 2232 for ( ; ; ) { 2233 /* check magic number -> version */ 2234 size_t const toRead = 4; 2235 const BYTE* const buf = (const BYTE*)ress.srcBuffer; 2236 if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ 2237 ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, 2238 (size_t)1, toRead - ress.srcBufferLoaded, srcFile); 2239 if (ress.srcBufferLoaded==0) { 2240 if (readSomething==0) { /* srcFile is empty (which is invalid) */ 2241 DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); 2242 return 1; 2243 } /* else, just reached frame boundary */ 2244 break; /* no more input */ 2245 } 2246 readSomething = 1; /* there is at least 1 byte in srcFile */ 2247 if (ress.srcBufferLoaded < toRead) { 2248 DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); 2249 return 1; 2250 } 2251 if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { 2252 unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, srcFileName, filesize); 2253 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2254 filesize += frameSize; 2255 } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ 2256 #ifdef ZSTD_GZDECOMPRESS 2257 unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, prefs, srcFileName); 2258 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2259 filesize += frameSize; 2260 #else 2261 DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); 2262 return 1; 2263 #endif 2264 } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ 2265 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ 2266 #ifdef ZSTD_LZMADECOMPRESS 2267 unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, prefs, srcFileName, buf[0] != 0xFD); 2268 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2269 filesize += frameSize; 2270 #else 2271 DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); 2272 return 1; 2273 #endif 2274 } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { 2275 #ifdef ZSTD_LZ4DECOMPRESS 2276 unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, prefs, srcFileName); 2277 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2278 filesize += frameSize; 2279 #else 2280 DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); 2281 return 1; 2282 #endif 2283 } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ 2284 return FIO_passThrough(prefs, 2285 ress.dstFile, srcFile, 2286 ress.srcBuffer, ress.srcBufferSize, 2287 ress.srcBufferLoaded); 2288 } else { 2289 DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); 2290 return 1; 2291 } } /* for each frame */ 2292 2293 /* Final Status */ 2294 DISPLAYLEVEL(2, "\r%79s\r", ""); 2295 DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); 2296 2297 return 0; 2298 } 2299 2300 /** FIO_decompressDstFile() : 2301 open `dstFileName`, 2302 or path-through if ress.dstFile is already != 0, 2303 then start decompression process (FIO_decompressFrames()). 2304 @return : 0 : OK 2305 1 : operation aborted 2306 */ 2307 static int FIO_decompressDstFile(FIO_prefs_t* const prefs, 2308 dRess_t ress, FILE* srcFile, 2309 const char* dstFileName, const char* srcFileName) 2310 { 2311 int result; 2312 stat_t statbuf; 2313 int transfer_permissions = 0; 2314 int releaseDstFile = 0; 2315 2316 if ((ress.dstFile == NULL) && (prefs->testMode==0)) { 2317 releaseDstFile = 1; 2318 2319 ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); 2320 if (ress.dstFile==NULL) return 1; 2321 2322 /* Must only be added after FIO_openDstFile() succeeds. 2323 * Otherwise we may delete the destination file if it already exists, 2324 * and the user presses Ctrl-C when asked if they wish to overwrite. 2325 */ 2326 addHandler(dstFileName); 2327 2328 if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ 2329 && UTIL_getFileStat(srcFileName, &statbuf) ) 2330 transfer_permissions = 1; 2331 } 2332 2333 result = FIO_decompressFrames(ress, srcFile, prefs, dstFileName, srcFileName); 2334 2335 if (releaseDstFile) { 2336 FILE* const dstFile = ress.dstFile; 2337 clearHandler(); 2338 ress.dstFile = NULL; 2339 if (fclose(dstFile)) { 2340 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 2341 result = 1; 2342 } 2343 2344 if ( (result != 0) /* operation failure */ 2345 && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ 2346 && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ 2347 ) { 2348 FIO_remove(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ 2349 } else { /* operation success */ 2350 if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */ 2351 && strcmp(dstFileName, nulmark) /* special case : don't chmod /dev/null */ 2352 && transfer_permissions ) /* file permissions correctly extracted from src */ 2353 UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ 2354 } 2355 } 2356 2357 return result; 2358 } 2359 2360 2361 /** FIO_decompressSrcFile() : 2362 Open `srcFileName`, transfer control to decompressDstFile() 2363 @return : 0 : OK 2364 1 : error 2365 */ 2366 static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) 2367 { 2368 FILE* srcFile; 2369 int result; 2370 2371 if (UTIL_isDirectory(srcFileName)) { 2372 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 2373 return 1; 2374 } 2375 2376 srcFile = FIO_openSrcFile(srcFileName); 2377 if (srcFile==NULL) return 1; 2378 ress.srcBufferLoaded = 0; 2379 2380 result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName); 2381 2382 /* Close file */ 2383 if (fclose(srcFile)) { 2384 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ 2385 return 1; 2386 } 2387 if ( prefs->removeSrcFile /* --rm */ 2388 && (result==0) /* decompression successful */ 2389 && strcmp(srcFileName, stdinmark) ) /* not stdin */ { 2390 /* We must clear the handler, since after this point calling it would 2391 * delete both the source and destination files. 2392 */ 2393 clearHandler(); 2394 if (FIO_remove(srcFileName)) { 2395 /* failed to remove src file */ 2396 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 2397 return 1; 2398 } } 2399 return result; 2400 } 2401 2402 2403 2404 int FIO_decompressFilename(FIO_prefs_t* const prefs, 2405 const char* dstFileName, const char* srcFileName, 2406 const char* dictFileName) 2407 { 2408 dRess_t const ress = FIO_createDResources(prefs, dictFileName); 2409 2410 int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); 2411 2412 FIO_freeDResources(ress); 2413 return decodingError; 2414 } 2415 2416 static const char *suffixList[] = { 2417 ZSTD_EXTENSION, 2418 TZSTD_EXTENSION, 2419 #ifdef ZSTD_GZDECOMPRESS 2420 GZ_EXTENSION, 2421 TGZ_EXTENSION, 2422 #endif 2423 #ifdef ZSTD_LZMADECOMPRESS 2424 LZMA_EXTENSION, 2425 XZ_EXTENSION, 2426 TXZ_EXTENSION, 2427 #endif 2428 #ifdef ZSTD_LZ4DECOMPRESS 2429 LZ4_EXTENSION, 2430 TLZ4_EXTENSION, 2431 #endif 2432 NULL 2433 }; 2434 2435 static const char *suffixListStr = 2436 ZSTD_EXTENSION "/" TZSTD_EXTENSION 2437 #ifdef ZSTD_GZDECOMPRESS 2438 "/" GZ_EXTENSION "/" TGZ_EXTENSION 2439 #endif 2440 #ifdef ZSTD_LZMADECOMPRESS 2441 "/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION 2442 #endif 2443 #ifdef ZSTD_LZ4DECOMPRESS 2444 "/" LZ4_EXTENSION "/" TLZ4_EXTENSION 2445 #endif 2446 ; 2447 2448 /* FIO_determineDstName() : 2449 * create a destination filename from a srcFileName. 2450 * @return a pointer to it. 2451 * @return == NULL if there is an error */ 2452 static const char* 2453 FIO_determineDstName(const char* srcFileName, const char* outDirName) 2454 { 2455 static size_t dfnbCapacity = 0; 2456 static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ 2457 size_t dstFileNameEndPos; 2458 char* outDirFilename = NULL; 2459 const char* dstSuffix = ""; 2460 size_t dstSuffixLen = 0; 2461 2462 size_t sfnSize = strlen(srcFileName); 2463 2464 size_t srcSuffixLen; 2465 const char* const srcSuffix = strrchr(srcFileName, '.'); 2466 if (srcSuffix == NULL) { 2467 DISPLAYLEVEL(1, 2468 "zstd: %s: unknown suffix (%s expected). " 2469 "Can't derive the output file name. " 2470 "Specify it with -o dstFileName. Ignoring.\n", 2471 srcFileName, suffixListStr); 2472 return NULL; 2473 } 2474 srcSuffixLen = strlen(srcSuffix); 2475 2476 { 2477 const char** matchedSuffixPtr; 2478 for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) { 2479 if (!strcmp(*matchedSuffixPtr, srcSuffix)) { 2480 break; 2481 } 2482 } 2483 2484 /* check suffix is authorized */ 2485 if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) { 2486 DISPLAYLEVEL(1, 2487 "zstd: %s: unknown suffix (%s expected). " 2488 "Can't derive the output file name. " 2489 "Specify it with -o dstFileName. Ignoring.\n", 2490 srcFileName, suffixListStr); 2491 return NULL; 2492 } 2493 2494 if ((*matchedSuffixPtr)[1] == 't') { 2495 dstSuffix = ".tar"; 2496 dstSuffixLen = strlen(dstSuffix); 2497 } 2498 } 2499 2500 if (outDirName) { 2501 outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); 2502 sfnSize = strlen(outDirFilename); 2503 assert(outDirFilename != NULL); 2504 } 2505 2506 if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) { 2507 /* allocate enough space to write dstFilename into it */ 2508 free(dstFileNameBuffer); 2509 dfnbCapacity = sfnSize + 20; 2510 dstFileNameBuffer = (char*)malloc(dfnbCapacity); 2511 if (dstFileNameBuffer==NULL) 2512 EXM_THROW(74, "%s : not enough memory for dstFileName", 2513 strerror(errno)); 2514 } 2515 2516 /* return dst name == src name truncated from suffix */ 2517 assert(dstFileNameBuffer != NULL); 2518 dstFileNameEndPos = sfnSize - srcSuffixLen; 2519 if (outDirFilename) { 2520 memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos); 2521 free(outDirFilename); 2522 } else { 2523 memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); 2524 } 2525 2526 /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" 2527 * extension on decompression. Also writes terminating null. */ 2528 strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix); 2529 return dstFileNameBuffer; 2530 2531 /* note : dstFileNameBuffer memory is not going to be free */ 2532 } 2533 2534 2535 int 2536 FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, 2537 const char** srcNamesTable, unsigned nbFiles, 2538 const char* outDirName, const char* outFileName, 2539 const char* dictFileName) 2540 { 2541 int error = 0; 2542 dRess_t ress = FIO_createDResources(prefs, dictFileName); 2543 2544 if (outFileName) { 2545 unsigned u; 2546 if (!prefs->testMode) { 2547 ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); 2548 if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); 2549 } 2550 for (u=0; u<nbFiles; u++) 2551 error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]); 2552 if ((!prefs->testMode) && (fclose(ress.dstFile))) 2553 EXM_THROW(72, "Write error : %s : cannot properly close output file", 2554 strerror(errno)); 2555 } else { 2556 unsigned u; 2557 for (u=0; u<nbFiles; u++) { /* create dstFileName */ 2558 const char* const srcFileName = srcNamesTable[u]; 2559 const char* const dstFileName = FIO_determineDstName(srcFileName, outDirName); 2560 if (dstFileName == NULL) { error=1; continue; } 2561 2562 error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); 2563 } 2564 if (outDirName) 2565 FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); 2566 } 2567 2568 FIO_freeDResources(ress); 2569 return error; 2570 } 2571 2572 /* ************************************************************************** 2573 * .zst file info (--list command) 2574 ***************************************************************************/ 2575 2576 typedef struct { 2577 U64 decompressedSize; 2578 U64 compressedSize; 2579 U64 windowSize; 2580 int numActualFrames; 2581 int numSkippableFrames; 2582 int decompUnavailable; 2583 int usesCheck; 2584 U32 nbFiles; 2585 } fileInfo_t; 2586 2587 typedef enum { 2588 info_success=0, 2589 info_frame_error=1, 2590 info_not_zstd=2, 2591 info_file_error=3, 2592 info_truncated_input=4, 2593 } InfoError; 2594 2595 #define ERROR_IF(c,n,...) { \ 2596 if (c) { \ 2597 DISPLAYLEVEL(1, __VA_ARGS__); \ 2598 DISPLAYLEVEL(1, " \n"); \ 2599 return n; \ 2600 } \ 2601 } 2602 2603 static InfoError 2604 FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) 2605 { 2606 /* begin analyzing frame */ 2607 for ( ; ; ) { 2608 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; 2609 size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); 2610 if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) { 2611 if ( feof(srcFile) 2612 && (numBytesRead == 0) 2613 && (info->compressedSize > 0) 2614 && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { 2615 unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); 2616 unsigned long long file_size = (unsigned long long) info->compressedSize; 2617 ERROR_IF(file_position != file_size, info_truncated_input, 2618 "Error: seeked to position %llu, which is beyond file size of %llu\n", 2619 file_position, 2620 file_size); 2621 break; /* correct end of file => success */ 2622 } 2623 ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); 2624 ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); 2625 } 2626 { U32 const magicNumber = MEM_readLE32(headerBuffer); 2627 /* Zstandard frame */ 2628 if (magicNumber == ZSTD_MAGICNUMBER) { 2629 ZSTD_frameHeader header; 2630 U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); 2631 if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR 2632 || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { 2633 info->decompUnavailable = 1; 2634 } else { 2635 info->decompressedSize += frameContentSize; 2636 } 2637 ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, 2638 info_frame_error, "Error: could not decode frame header"); 2639 info->windowSize = header.windowSize; 2640 /* move to the end of the frame header */ 2641 { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); 2642 ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); 2643 ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, 2644 info_frame_error, "Error: could not move to end of frame header"); 2645 } 2646 2647 /* skip all blocks in the frame */ 2648 { int lastBlock = 0; 2649 do { 2650 BYTE blockHeaderBuffer[3]; 2651 ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, 2652 info_frame_error, "Error while reading block header"); 2653 { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); 2654 U32 const blockTypeID = (blockHeader >> 1) & 3; 2655 U32 const isRLE = (blockTypeID == 1); 2656 U32 const isWrongBlock = (blockTypeID == 3); 2657 long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); 2658 ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); 2659 lastBlock = blockHeader & 1; 2660 ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, 2661 info_frame_error, "Error: could not skip to end of block"); 2662 } 2663 } while (lastBlock != 1); 2664 } 2665 2666 /* check if checksum is used */ 2667 { BYTE const frameHeaderDescriptor = headerBuffer[4]; 2668 int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; 2669 if (contentChecksumFlag) { 2670 info->usesCheck = 1; 2671 ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, 2672 info_frame_error, "Error: could not skip past checksum"); 2673 } } 2674 info->numActualFrames++; 2675 } 2676 /* Skippable frame */ 2677 else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 2678 U32 const frameSize = MEM_readLE32(headerBuffer + 4); 2679 long const seek = (long)(8 + frameSize - numBytesRead); 2680 ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, 2681 info_frame_error, "Error: could not find end of skippable frame"); 2682 info->numSkippableFrames++; 2683 } 2684 /* unknown content */ 2685 else { 2686 return info_not_zstd; 2687 } 2688 } /* magic number analysis */ 2689 } /* end analyzing frames */ 2690 return info_success; 2691 } 2692 2693 2694 static InfoError 2695 getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) 2696 { 2697 InfoError status; 2698 FILE* const srcFile = FIO_openSrcFile(inFileName); 2699 ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); 2700 2701 info->compressedSize = UTIL_getFileSize(inFileName); 2702 status = FIO_analyzeFrames(info, srcFile); 2703 2704 fclose(srcFile); 2705 info->nbFiles = 1; 2706 return status; 2707 } 2708 2709 2710 /** getFileInfo() : 2711 * Reads information from file, stores in *info 2712 * @return : InfoError status 2713 */ 2714 static InfoError 2715 getFileInfo(fileInfo_t* info, const char* srcFileName) 2716 { 2717 ERROR_IF(!UTIL_isRegularFile(srcFileName), 2718 info_file_error, "Error : %s is not a file", srcFileName); 2719 return getFileInfo_fileConfirmed(info, srcFileName); 2720 } 2721 2722 2723 static void 2724 displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) 2725 { 2726 unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); 2727 const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; 2728 double const windowSizeUnit = (double)info->windowSize / unit; 2729 double const compressedSizeUnit = (double)info->compressedSize / unit; 2730 double const decompressedSizeUnit = (double)info->decompressedSize / unit; 2731 double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; 2732 const char* const checkString = (info->usesCheck ? "XXH64" : "None"); 2733 if (displayLevel <= 2) { 2734 if (!info->decompUnavailable) { 2735 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", 2736 info->numSkippableFrames + info->numActualFrames, 2737 info->numSkippableFrames, 2738 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 2739 ratio, checkString, inFileName); 2740 } else { 2741 DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n", 2742 info->numSkippableFrames + info->numActualFrames, 2743 info->numSkippableFrames, 2744 compressedSizeUnit, unitStr, 2745 checkString, inFileName); 2746 } 2747 } else { 2748 DISPLAYOUT("%s \n", inFileName); 2749 DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); 2750 if (info->numSkippableFrames) 2751 DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); 2752 DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", 2753 windowSizeUnit, unitStr, 2754 (unsigned long long)info->windowSize); 2755 DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", 2756 compressedSizeUnit, unitStr, 2757 (unsigned long long)info->compressedSize); 2758 if (!info->decompUnavailable) { 2759 DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", 2760 decompressedSizeUnit, unitStr, 2761 (unsigned long long)info->decompressedSize); 2762 DISPLAYOUT("Ratio: %.4f\n", ratio); 2763 } 2764 DISPLAYOUT("Check: %s\n", checkString); 2765 DISPLAYOUT("\n"); 2766 } 2767 } 2768 2769 static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) 2770 { 2771 fileInfo_t total; 2772 memset(&total, 0, sizeof(total)); 2773 total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; 2774 total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; 2775 total.compressedSize = fi1.compressedSize + fi2.compressedSize; 2776 total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; 2777 total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; 2778 total.usesCheck = fi1.usesCheck & fi2.usesCheck; 2779 total.nbFiles = fi1.nbFiles + fi2.nbFiles; 2780 return total; 2781 } 2782 2783 static int 2784 FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) 2785 { 2786 fileInfo_t info; 2787 memset(&info, 0, sizeof(info)); 2788 { InfoError const error = getFileInfo(&info, inFileName); 2789 switch (error) { 2790 case info_frame_error: 2791 /* display error, but provide output */ 2792 DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName); 2793 break; 2794 case info_not_zstd: 2795 DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName); 2796 if (displayLevel > 2) DISPLAYOUT("\n"); 2797 return 1; 2798 case info_file_error: 2799 /* error occurred while opening the file */ 2800 if (displayLevel > 2) DISPLAYOUT("\n"); 2801 return 1; 2802 case info_truncated_input: 2803 DISPLAYOUT("File \"%s\" is truncated \n", inFileName); 2804 if (displayLevel > 2) DISPLAYOUT("\n"); 2805 return 1; 2806 case info_success: 2807 default: 2808 break; 2809 } 2810 2811 displayInfo(inFileName, &info, displayLevel); 2812 *total = FIO_addFInfo(*total, info); 2813 assert(error == info_success || error == info_frame_error); 2814 return (int)error; 2815 } 2816 } 2817 2818 int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel) 2819 { 2820 /* ensure no specified input is stdin (needs fseek() capability) */ 2821 { unsigned u; 2822 for (u=0; u<numFiles;u++) { 2823 ERROR_IF(!strcmp (filenameTable[u], stdinmark), 2824 1, "zstd: --list does not support reading from standard input"); 2825 } } 2826 2827 if (numFiles == 0) { 2828 if (!IS_CONSOLE(stdin)) { 2829 DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n"); 2830 } 2831 DISPLAYLEVEL(1, "No files given \n"); 2832 return 1; 2833 } 2834 2835 if (displayLevel <= 2) { 2836 DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); 2837 } 2838 { int error = 0; 2839 fileInfo_t total; 2840 memset(&total, 0, sizeof(total)); 2841 total.usesCheck = 1; 2842 /* --list each file, and check for any error */ 2843 { unsigned u; 2844 for (u=0; u<numFiles;u++) { 2845 error |= FIO_listFile(&total, filenameTable[u], displayLevel); 2846 } } 2847 if (numFiles > 1 && displayLevel <= 2) { /* display total */ 2848 unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); 2849 const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; 2850 double const compressedSizeUnit = (double)total.compressedSize / unit; 2851 double const decompressedSizeUnit = (double)total.decompressedSize / unit; 2852 double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize; 2853 const char* const checkString = (total.usesCheck ? "XXH64" : ""); 2854 DISPLAYOUT("----------------------------------------------------------------- \n"); 2855 if (total.decompUnavailable) { 2856 DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n", 2857 total.numSkippableFrames + total.numActualFrames, 2858 total.numSkippableFrames, 2859 compressedSizeUnit, unitStr, 2860 checkString, (unsigned)total.nbFiles); 2861 } else { 2862 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n", 2863 total.numSkippableFrames + total.numActualFrames, 2864 total.numSkippableFrames, 2865 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 2866 ratio, checkString, (unsigned)total.nbFiles); 2867 } } 2868 return error; 2869 } 2870 } 2871 2872 2873 #endif /* #ifndef ZSTD_NODECOMPRESS */ 2874