1 /* 2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /* ************************************* 13 * Compiler Options 14 ***************************************/ 15 #ifdef _MSC_VER /* Visual */ 16 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 17 # pragma warning(disable : 4204) /* non-constant aggregate initializer */ 18 #endif 19 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) 20 # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ 21 #endif 22 23 24 /*-************************************* 25 * Includes 26 ***************************************/ 27 #include "platform.h" /* Large Files support, SET_BINARY_MODE */ 28 #include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile */ 29 #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ 30 #include <stdlib.h> /* malloc, free */ 31 #include <string.h> /* strcmp, strlen */ 32 #include <errno.h> /* errno */ 33 34 #if defined (_MSC_VER) 35 # include <sys/stat.h> 36 # include <io.h> 37 #endif 38 39 #include "bitstream.h" 40 #include "mem.h" 41 #include "fileio.h" 42 #include "util.h" 43 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ 44 #include "zstd.h" 45 #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) 46 # include <zlib.h> 47 # if !defined(z_const) 48 # define z_const 49 # endif 50 #endif 51 #if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) 52 # include <lzma.h> 53 #endif 54 55 #define LZ4_MAGICNUMBER 0x184D2204 56 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) 57 # define LZ4F_ENABLE_OBSOLETE_ENUMS 58 # include <lz4frame.h> 59 # include <lz4.h> 60 #endif 61 62 63 /*-************************************* 64 * Constants 65 ***************************************/ 66 #define KB *(1<<10) 67 #define MB *(1<<20) 68 #define GB *(1U<<30) 69 70 #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ 71 72 #define FNSPACE 30 73 74 75 /*-************************************* 76 * Macros 77 ***************************************/ 78 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 79 #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) 80 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } 81 static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ 82 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } 83 84 static const U64 g_refreshRate = SEC_TO_MICRO / 6; 85 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; 86 87 #define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \ 88 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ 89 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ 90 if (g_displayLevel>=4) fflush(stderr); } } } 91 92 #undef MIN /* in case it would be already defined */ 93 #define MIN(a,b) ((a) < (b) ? (a) : (b)) 94 95 96 /*-************************************* 97 * Debug 98 ***************************************/ 99 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) 100 # include <assert.h> 101 #else 102 # ifndef assert 103 # define assert(condition) ((void)0) 104 # endif 105 #endif 106 107 #ifndef ZSTD_DEBUG 108 # define ZSTD_DEBUG 0 109 #endif 110 #define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__); 111 #define EXM_THROW(error, ...) \ 112 { \ 113 DISPLAYLEVEL(1, "zstd: "); \ 114 DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 115 DISPLAYLEVEL(1, "error %i : ", error); \ 116 DISPLAYLEVEL(1, __VA_ARGS__); \ 117 DISPLAYLEVEL(1, " \n"); \ 118 exit(error); \ 119 } 120 121 #define CHECK_V(v, f) \ 122 v = f; \ 123 if (ZSTD_isError(v)) { \ 124 DEBUGLOG(1, "%s \n", #f); \ 125 EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ 126 } 127 #define CHECK(f) { size_t err; CHECK_V(err, f); } 128 129 130 /*-************************************ 131 * Signal (Ctrl-C trapping) 132 **************************************/ 133 #include <signal.h> 134 135 static const char* g_artefact = NULL; 136 static void INThandler(int sig) 137 { 138 assert(sig==SIGINT); (void)sig; 139 #if !defined(_MSC_VER) 140 signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ 141 #endif 142 if (g_artefact) remove(g_artefact); 143 DISPLAY("\n"); 144 exit(2); 145 } 146 static void addHandler(char const* dstFileName) 147 { 148 if (UTIL_isRegularFile(dstFileName)) { 149 g_artefact = dstFileName; 150 signal(SIGINT, INThandler); 151 } else { 152 g_artefact = NULL; 153 } 154 } 155 /* Idempotent */ 156 static void clearHandler(void) 157 { 158 if (g_artefact) signal(SIGINT, SIG_DFL); 159 g_artefact = NULL; 160 } 161 162 163 /* ************************************************************ 164 * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW 165 ***************************************************************/ 166 #if defined(_MSC_VER) && _MSC_VER >= 1400 167 # define LONG_SEEK _fseeki64 168 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 169 # define LONG_SEEK fseeko 170 #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) 171 # define LONG_SEEK fseeko64 172 #elif defined(_WIN32) && !defined(__DJGPP__) 173 # include <windows.h> 174 static int LONG_SEEK(FILE* file, __int64 offset, int origin) { 175 LARGE_INTEGER off; 176 DWORD method; 177 off.QuadPart = offset; 178 if (origin == SEEK_END) 179 method = FILE_END; 180 else if (origin == SEEK_CUR) 181 method = FILE_CURRENT; 182 else 183 method = FILE_BEGIN; 184 185 if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) 186 return 0; 187 else 188 return -1; 189 } 190 #else 191 # define LONG_SEEK fseek 192 #endif 193 194 195 /*-************************************* 196 * Local Parameters - Not thread safe 197 ***************************************/ 198 static FIO_compressionType_t g_compressionType = FIO_zstdCompression; 199 void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } 200 static U32 g_overwrite = 0; 201 void FIO_overwriteMode(void) { g_overwrite=1; } 202 static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ 203 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } 204 static U32 g_dictIDFlag = 1; 205 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } 206 static U32 g_checksumFlag = 1; 207 void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } 208 static U32 g_removeSrcFile = 0; 209 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } 210 static U32 g_memLimit = 0; 211 void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } 212 static U32 g_nbThreads = 1; 213 void FIO_setNbThreads(unsigned nbThreads) { 214 #ifndef ZSTD_MULTITHREAD 215 if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); 216 #endif 217 g_nbThreads = nbThreads; 218 } 219 static U32 g_blockSize = 0; 220 void FIO_setBlockSize(unsigned blockSize) { 221 if (blockSize && g_nbThreads==1) 222 DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); 223 g_blockSize = blockSize; 224 } 225 #define FIO_OVERLAP_LOG_NOTSET 9999 226 static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET; 227 void FIO_setOverlapLog(unsigned overlapLog){ 228 if (overlapLog && g_nbThreads==1) 229 DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); 230 g_overlapLog = overlapLog; 231 } 232 static U32 g_ldmFlag = 0; 233 void FIO_setLdmFlag(unsigned ldmFlag) { 234 g_ldmFlag = (ldmFlag>0); 235 } 236 static U32 g_ldmHashLog = 0; 237 void FIO_setLdmHashLog(unsigned ldmHashLog) { 238 g_ldmHashLog = ldmHashLog; 239 } 240 static U32 g_ldmMinMatch = 0; 241 void FIO_setLdmMinMatch(unsigned ldmMinMatch) { 242 g_ldmMinMatch = ldmMinMatch; 243 } 244 245 #define FIO_LDM_PARAM_NOTSET 9999 246 static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; 247 void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) { 248 g_ldmBucketSizeLog = ldmBucketSizeLog; 249 } 250 251 static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET; 252 void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) { 253 g_ldmHashEveryLog = ldmHashEveryLog; 254 } 255 256 257 258 /*-************************************* 259 * Functions 260 ***************************************/ 261 /** FIO_remove() : 262 * @result : Unlink `fileName`, even if it's read-only */ 263 static int FIO_remove(const char* path) 264 { 265 if (!UTIL_isRegularFile(path)) { 266 DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path); 267 return 0; 268 } 269 #if defined(_WIN32) || defined(WIN32) 270 /* windows doesn't allow remove read-only files, 271 * so try to make it writable first */ 272 chmod(path, _S_IWRITE); 273 #endif 274 return remove(path); 275 } 276 277 /** FIO_openSrcFile() : 278 * condition : `srcFileName` must be non-NULL. 279 * @result : FILE* to `srcFileName`, or NULL if it fails */ 280 static FILE* FIO_openSrcFile(const char* srcFileName) 281 { 282 assert(srcFileName != NULL); 283 if (!strcmp (srcFileName, stdinmark)) { 284 DISPLAYLEVEL(4,"Using stdin for input\n"); 285 SET_BINARY_MODE(stdin); 286 return stdin; 287 } 288 289 if (!UTIL_isRegularFile(srcFileName)) { 290 DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", 291 srcFileName); 292 return NULL; 293 } 294 295 { FILE* const f = fopen(srcFileName, "rb"); 296 if (f == NULL) 297 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 298 return f; 299 } 300 } 301 302 /** FIO_openDstFile() : 303 * condition : `dstFileName` must be non-NULL. 304 * @result : FILE* to `dstFileName`, or NULL if it fails */ 305 static FILE* FIO_openDstFile(const char* dstFileName) 306 { 307 assert(dstFileName != NULL); 308 if (!strcmp (dstFileName, stdoutmark)) { 309 DISPLAYLEVEL(4,"Using stdout for output\n"); 310 SET_BINARY_MODE(stdout); 311 if (g_sparseFileSupport==1) { 312 g_sparseFileSupport = 0; 313 DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 314 } 315 return stdout; 316 } 317 318 if (g_sparseFileSupport == 1) { 319 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; 320 } 321 322 if (UTIL_isRegularFile(dstFileName)) { 323 FILE* fCheck; 324 if (!strcmp(dstFileName, nulmark)) { 325 EXM_THROW(40, "%s is unexpectedly a regular file", dstFileName); 326 } 327 /* Check if destination file already exists */ 328 fCheck = fopen( dstFileName, "rb" ); 329 if (fCheck != NULL) { /* dst file exists, authorization prompt */ 330 fclose(fCheck); 331 if (!g_overwrite) { 332 if (g_displayLevel <= 1) { 333 /* No interaction possible */ 334 DISPLAY("zstd: %s already exists; not overwritten \n", 335 dstFileName); 336 return NULL; 337 } 338 DISPLAY("zstd: %s already exists; overwrite (y/N) ? ", 339 dstFileName); 340 { int ch = getchar(); 341 if ((ch!='Y') && (ch!='y')) { 342 DISPLAY(" not overwritten \n"); 343 return NULL; 344 } 345 /* flush rest of input line */ 346 while ((ch!=EOF) && (ch!='\n')) ch = getchar(); 347 } } 348 /* need to unlink */ 349 FIO_remove(dstFileName); 350 } } 351 352 { FILE* const f = fopen( dstFileName, "wb" ); 353 if (f == NULL) 354 DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); 355 return f; 356 } 357 } 358 359 360 /*! FIO_createDictBuffer() : 361 * creates a buffer, pointed by `*bufferPtr`, 362 * loads `filename` content into it, up to DICTSIZE_MAX bytes. 363 * @return : loaded size 364 * if fileName==NULL, returns 0 and a NULL pointer 365 */ 366 static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) 367 { 368 FILE* fileHandle; 369 U64 fileSize; 370 371 assert(bufferPtr != NULL); 372 *bufferPtr = NULL; 373 if (fileName == NULL) return 0; 374 375 DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); 376 fileHandle = fopen(fileName, "rb"); 377 if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); 378 fileSize = UTIL_getFileSize(fileName); 379 if (fileSize > DICTSIZE_MAX) { 380 EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", 381 fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */ 382 } 383 *bufferPtr = malloc((size_t)fileSize); 384 if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); 385 { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); 386 if (readSize!=fileSize) 387 EXM_THROW(35, "Error reading dictionary file %s", fileName); 388 } 389 fclose(fileHandle); 390 return (size_t)fileSize; 391 } 392 393 #ifndef ZSTD_NOCOMPRESS 394 395 /*-********************************************************************** 396 * Compression 397 ************************************************************************/ 398 typedef struct { 399 FILE* srcFile; 400 FILE* dstFile; 401 void* srcBuffer; 402 size_t srcBufferSize; 403 void* dstBuffer; 404 size_t dstBufferSize; 405 ZSTD_CStream* cctx; 406 } cRess_t; 407 408 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, 409 U64 srcSize, 410 ZSTD_compressionParameters* comprParams) { 411 cRess_t ress; 412 memset(&ress, 0, sizeof(ress)); 413 414 ress.cctx = ZSTD_createCCtx(); 415 if (ress.cctx == NULL) 416 EXM_THROW(30, "allocation error : can't create ZSTD_CCtx"); 417 ress.srcBufferSize = ZSTD_CStreamInSize(); 418 ress.srcBuffer = malloc(ress.srcBufferSize); 419 ress.dstBufferSize = ZSTD_CStreamOutSize(); 420 ress.dstBuffer = malloc(ress.dstBufferSize); 421 if (!ress.srcBuffer || !ress.dstBuffer) 422 EXM_THROW(31, "allocation error : not enough memory"); 423 424 /* Advances parameters, including dictionary */ 425 { void* dictBuffer; 426 size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ 427 if (dictFileName && (dictBuffer==NULL)) 428 EXM_THROW(32, "allocation error : can't create dictBuffer"); 429 430 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */ 431 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); 432 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); 433 /* compression level */ 434 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); 435 /* long distance matching */ 436 CHECK( ZSTD_CCtx_setParameter( 437 ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) ); 438 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) ); 439 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) ); 440 if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { 441 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) ); 442 } 443 if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) { 444 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) ); 445 } 446 /* compression parameters */ 447 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) ); 448 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) ); 449 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) ); 450 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) ); 451 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) ); 452 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); 453 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); 454 /* multi-threading */ 455 DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); 456 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); 457 /* dictionary */ 458 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, for compression parameters adaptation */ 459 CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); 460 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ 461 462 free(dictBuffer); 463 } 464 465 return ress; 466 } 467 468 static void FIO_freeCResources(cRess_t ress) 469 { 470 free(ress.srcBuffer); 471 free(ress.dstBuffer); 472 ZSTD_freeCStream(ress.cctx); /* never fails */ 473 } 474 475 476 #ifdef ZSTD_GZCOMPRESS 477 static unsigned long long FIO_compressGzFrame(cRess_t* ress, 478 const char* srcFileName, U64 const srcFileSize, 479 int compressionLevel, U64* readsize) 480 { 481 unsigned long long inFileSize = 0, outFileSize = 0; 482 z_stream strm; 483 int ret; 484 485 if (compressionLevel > Z_BEST_COMPRESSION) 486 compressionLevel = Z_BEST_COMPRESSION; 487 488 strm.zalloc = Z_NULL; 489 strm.zfree = Z_NULL; 490 strm.opaque = Z_NULL; 491 492 ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 493 15 /* maxWindowLogSize */ + 16 /* gzip only */, 494 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 495 if (ret != Z_OK) 496 EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); 497 498 strm.next_in = 0; 499 strm.avail_in = 0; 500 strm.next_out = (Bytef*)ress->dstBuffer; 501 strm.avail_out = (uInt)ress->dstBufferSize; 502 503 while (1) { 504 if (strm.avail_in == 0) { 505 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 506 if (inSize == 0) break; 507 inFileSize += inSize; 508 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 509 strm.avail_in = (uInt)inSize; 510 } 511 ret = deflate(&strm, Z_NO_FLUSH); 512 if (ret != Z_OK) 513 EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); 514 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 515 if (decompBytes) { 516 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) 517 EXM_THROW(73, "Write error : cannot write to output file"); 518 outFileSize += decompBytes; 519 strm.next_out = (Bytef*)ress->dstBuffer; 520 strm.avail_out = (uInt)ress->dstBufferSize; 521 } 522 } 523 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 524 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 525 (U32)(inFileSize>>20), 526 (double)outFileSize/inFileSize*100) 527 else 528 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 529 (U32)(inFileSize>>20), (U32)(srcFileSize>>20), 530 (double)outFileSize/inFileSize*100); 531 } 532 533 while (1) { 534 ret = deflate(&strm, Z_FINISH); 535 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 536 if (decompBytes) { 537 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) 538 EXM_THROW(75, "Write error : cannot write to output file"); 539 outFileSize += decompBytes; 540 strm.next_out = (Bytef*)ress->dstBuffer; 541 strm.avail_out = (uInt)ress->dstBufferSize; 542 } } 543 if (ret == Z_STREAM_END) break; 544 if (ret != Z_BUF_ERROR) 545 EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); 546 } 547 548 ret = deflateEnd(&strm); 549 if (ret != Z_OK) 550 EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); 551 *readsize = inFileSize; 552 553 return outFileSize; 554 } 555 #endif 556 557 558 #ifdef ZSTD_LZMACOMPRESS 559 static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, 560 const char* srcFileName, U64 const srcFileSize, 561 int compressionLevel, U64* readsize, int plain_lzma) 562 { 563 unsigned long long inFileSize = 0, outFileSize = 0; 564 lzma_stream strm = LZMA_STREAM_INIT; 565 lzma_action action = LZMA_RUN; 566 lzma_ret ret; 567 568 if (compressionLevel < 0) compressionLevel = 0; 569 if (compressionLevel > 9) compressionLevel = 9; 570 571 if (plain_lzma) { 572 lzma_options_lzma opt_lzma; 573 if (lzma_lzma_preset(&opt_lzma, compressionLevel)) 574 EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); 575 ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ 576 if (ret != LZMA_OK) 577 EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); 578 } else { 579 ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ 580 if (ret != LZMA_OK) 581 EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); 582 } 583 584 strm.next_in = 0; 585 strm.avail_in = 0; 586 strm.next_out = (BYTE*)ress->dstBuffer; 587 strm.avail_out = ress->dstBufferSize; 588 589 while (1) { 590 if (strm.avail_in == 0) { 591 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 592 if (inSize == 0) action = LZMA_FINISH; 593 inFileSize += inSize; 594 strm.next_in = (BYTE const*)ress->srcBuffer; 595 strm.avail_in = inSize; 596 } 597 598 ret = lzma_code(&strm, action); 599 600 if (ret != LZMA_OK && ret != LZMA_STREAM_END) 601 EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); 602 { size_t const compBytes = ress->dstBufferSize - strm.avail_out; 603 if (compBytes) { 604 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) 605 EXM_THROW(73, "Write error : cannot write to output file"); 606 outFileSize += compBytes; 607 strm.next_out = (BYTE*)ress->dstBuffer; 608 strm.avail_out = ress->dstBufferSize; 609 } } 610 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 611 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 612 (U32)(inFileSize>>20), 613 (double)outFileSize/inFileSize*100) 614 else 615 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 616 (U32)(inFileSize>>20), (U32)(srcFileSize>>20), 617 (double)outFileSize/inFileSize*100); 618 if (ret == LZMA_STREAM_END) break; 619 } 620 621 lzma_end(&strm); 622 *readsize = inFileSize; 623 624 return outFileSize; 625 } 626 #endif 627 628 #ifdef ZSTD_LZ4COMPRESS 629 #if LZ4_VERSION_NUMBER <= 10600 630 #define LZ4F_blockLinked blockLinked 631 #define LZ4F_max64KB max64KB 632 #endif 633 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 634 static unsigned long long FIO_compressLz4Frame(cRess_t* ress, 635 const char* srcFileName, U64 const srcFileSize, 636 int compressionLevel, U64* readsize) 637 { 638 const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); 639 unsigned long long inFileSize = 0, outFileSize = 0; 640 641 LZ4F_preferences_t prefs; 642 LZ4F_compressionContext_t ctx; 643 644 LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 645 if (LZ4F_isError(errorCode)) 646 EXM_THROW(31, "zstd: failed to create lz4 compression context"); 647 648 memset(&prefs, 0, sizeof(prefs)); 649 650 assert(blockSize <= ress->srcBufferSize); 651 652 prefs.autoFlush = 1; 653 prefs.compressionLevel = compressionLevel; 654 prefs.frameInfo.blockMode = LZ4F_blockLinked; 655 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 656 prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; 657 #if LZ4_VERSION_NUMBER >= 10600 658 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; 659 #endif 660 assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); 661 662 { 663 size_t readSize; 664 size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); 665 if (LZ4F_isError(headerSize)) 666 EXM_THROW(33, "File header generation failed : %s", 667 LZ4F_getErrorName(headerSize)); 668 if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) 669 EXM_THROW(34, "Write error : cannot write header"); 670 outFileSize += headerSize; 671 672 /* Read first block */ 673 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 674 inFileSize += readSize; 675 676 /* Main Loop */ 677 while (readSize>0) { 678 size_t outSize; 679 680 /* Compress Block */ 681 outSize = LZ4F_compressUpdate(ctx, ress->dstBuffer, ress->dstBufferSize, ress->srcBuffer, readSize, NULL); 682 if (LZ4F_isError(outSize)) 683 EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", 684 srcFileName, LZ4F_getErrorName(outSize)); 685 outFileSize += outSize; 686 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 687 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 688 (U32)(inFileSize>>20), 689 (double)outFileSize/inFileSize*100) 690 else 691 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 692 (U32)(inFileSize>>20), (U32)(srcFileSize>>20), 693 (double)outFileSize/inFileSize*100); 694 695 /* Write Block */ 696 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); 697 if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } 698 699 /* Read next block */ 700 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 701 inFileSize += readSize; 702 } 703 if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); 704 705 /* End of Stream mark */ 706 headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); 707 if (LZ4F_isError(headerSize)) 708 EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", 709 srcFileName, LZ4F_getErrorName(headerSize)); 710 711 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); 712 if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } 713 outFileSize += headerSize; 714 } 715 716 *readsize = inFileSize; 717 LZ4F_freeCompressionContext(ctx); 718 719 return outFileSize; 720 } 721 #endif 722 723 724 /*! FIO_compressFilename_internal() : 725 * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. 726 * @return : 0 : compression completed correctly, 727 * 1 : missing or pb opening srcFileName 728 */ 729 static int FIO_compressFilename_internal(cRess_t ress, 730 const char* dstFileName, const char* srcFileName, int compressionLevel) 731 { 732 FILE* const srcFile = ress.srcFile; 733 FILE* const dstFile = ress.dstFile; 734 U64 readsize = 0; 735 U64 compressedfilesize = 0; 736 U64 const fileSize = UTIL_getFileSize(srcFileName); 737 ZSTD_EndDirective directive = ZSTD_e_continue; 738 DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize); 739 740 switch (g_compressionType) { 741 case FIO_zstdCompression: 742 break; 743 744 case FIO_gzipCompression: 745 #ifdef ZSTD_GZCOMPRESS 746 compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 747 #else 748 (void)compressionLevel; 749 EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", 750 srcFileName); 751 #endif 752 goto finish; 753 754 case FIO_xzCompression: 755 case FIO_lzmaCompression: 756 #ifdef ZSTD_LZMACOMPRESS 757 compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); 758 #else 759 (void)compressionLevel; 760 EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", 761 srcFileName); 762 #endif 763 goto finish; 764 765 case FIO_lz4Compression: 766 #ifdef ZSTD_LZ4COMPRESS 767 compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 768 #else 769 (void)compressionLevel; 770 EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", 771 srcFileName); 772 #endif 773 goto finish; 774 } 775 776 /* init */ 777 if (fileSize != UTIL_FILESIZE_UNKNOWN) 778 ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); 779 780 /* Main compression loop */ 781 do { 782 size_t result; 783 /* Fill input Buffer */ 784 size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); 785 ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; 786 readsize += inSize; 787 788 if (inSize == 0 || (fileSize != UTIL_FILESIZE_UNKNOWN && readsize == fileSize)) 789 directive = ZSTD_e_end; 790 791 result = 1; 792 while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) { 793 ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; 794 CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); 795 796 /* Write compressed stream */ 797 DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_continue: generated %u bytes \n", 798 (U32)outBuff.pos); 799 if (outBuff.pos) { 800 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); 801 if (sizeCheck!=outBuff.pos) 802 EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); 803 compressedfilesize += outBuff.pos; 804 } 805 } 806 if (g_nbThreads > 1) { 807 if (fileSize == UTIL_FILESIZE_UNKNOWN) 808 DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20)) 809 else 810 DISPLAYUPDATE(2, "\rRead : %u / %u MB", 811 (U32)(readsize>>20), (U32)(fileSize>>20)); 812 } else { 813 if (fileSize == UTIL_FILESIZE_UNKNOWN) 814 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 815 (U32)(readsize>>20), 816 (double)compressedfilesize/readsize*100) 817 else 818 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 819 (U32)(readsize>>20), (U32)(fileSize>>20), 820 (double)compressedfilesize/readsize*100); 821 } 822 } while (directive != ZSTD_e_end); 823 824 finish: 825 /* Status */ 826 DISPLAYLEVEL(2, "\r%79s\r", ""); 827 DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, 828 (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100, 829 (unsigned long long)readsize, (unsigned long long) compressedfilesize, 830 dstFileName); 831 832 return 0; 833 } 834 835 836 /*! FIO_compressFilename_srcFile() : 837 * note : ress.destFile already opened 838 * @return : 0 : compression completed correctly, 839 * 1 : missing or pb opening srcFileName 840 */ 841 static int FIO_compressFilename_srcFile(cRess_t ress, 842 const char* dstFileName, const char* srcFileName, 843 int compressionLevel) 844 { 845 int result; 846 847 /* File check */ 848 if (UTIL_isDirectory(srcFileName)) { 849 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 850 return 1; 851 } 852 853 ress.srcFile = FIO_openSrcFile(srcFileName); 854 if (!ress.srcFile) return 1; /* srcFile could not be opened */ 855 856 result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); 857 858 fclose(ress.srcFile); 859 if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) { 860 /* We must clear the handler, since after this point calling it would 861 * delete both the source and destination files. 862 */ 863 clearHandler(); 864 if (remove(srcFileName)) 865 EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); 866 } 867 return result; 868 } 869 870 871 /*! FIO_compressFilename_dstFile() : 872 * @return : 0 : compression completed correctly, 873 * 1 : pb 874 */ 875 static int FIO_compressFilename_dstFile(cRess_t ress, 876 const char* dstFileName, 877 const char* srcFileName, 878 int compressionLevel) 879 { 880 int result; 881 stat_t statbuf; 882 int stat_result = 0; 883 884 ress.dstFile = FIO_openDstFile(dstFileName); 885 if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ 886 /* Must ony be added after FIO_openDstFile() succeeds. 887 * Otherwise we may delete the destination file if at already exists, and 888 * the user presses Ctrl-C when asked if they wish to overwrite. 889 */ 890 addHandler(dstFileName); 891 892 if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) 893 stat_result = 1; 894 result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); 895 clearHandler(); 896 897 if (fclose(ress.dstFile)) { /* error closing dstFile */ 898 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 899 result=1; 900 } 901 if (result!=0) { /* remove operation artefact */ 902 if (remove(dstFileName)) 903 EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); 904 } 905 else if (strcmp (dstFileName, stdoutmark) && stat_result) 906 UTIL_setFileStat(dstFileName, &statbuf); 907 908 return result; 909 } 910 911 912 int FIO_compressFilename(const char* dstFileName, const char* srcFileName, 913 const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams) 914 { 915 clock_t const start = clock(); 916 U64 const fileSize = UTIL_getFileSize(srcFileName); 917 U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; 918 919 cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); 920 int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); 921 922 double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; 923 DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); 924 925 FIO_freeCResources(ress); 926 return result; 927 } 928 929 930 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, 931 const char* outFileName, const char* suffix, 932 const char* dictFileName, int compressionLevel, 933 ZSTD_compressionParameters* comprParams) 934 { 935 int missed_files = 0; 936 size_t dfnSize = FNSPACE; 937 char* dstFileName = (char*)malloc(FNSPACE); 938 size_t const suffixSize = suffix ? strlen(suffix) : 0; 939 U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); 940 U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; 941 U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ; 942 cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); 943 944 /* init */ 945 if (dstFileName==NULL) 946 EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); 947 if (outFileName == NULL && suffix == NULL) 948 EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown"); /* should never happen */ 949 950 /* loop on each file */ 951 if (outFileName != NULL) { 952 unsigned u; 953 ress.dstFile = FIO_openDstFile(outFileName); 954 for (u=0; u<nbFiles; u++) 955 missed_files += FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel); 956 if (fclose(ress.dstFile)) 957 EXM_THROW(29, "Write error : cannot properly close stdout"); 958 } else { 959 unsigned u; 960 for (u=0; u<nbFiles; u++) { 961 size_t const ifnSize = strlen(inFileNamesTable[u]); 962 if (dfnSize <= ifnSize+suffixSize+1) { /* resize name buffer */ 963 free(dstFileName); 964 dfnSize = ifnSize + 20; 965 dstFileName = (char*)malloc(dfnSize); 966 if (!dstFileName) { 967 EXM_THROW(30, "zstd: %s", strerror(errno)); 968 } } 969 strcpy(dstFileName, inFileNamesTable[u]); 970 strcat(dstFileName, suffix); 971 missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel); 972 } } 973 974 FIO_freeCResources(ress); 975 free(dstFileName); 976 return missed_files; 977 } 978 979 #endif /* #ifndef ZSTD_NOCOMPRESS */ 980 981 982 983 #ifndef ZSTD_NODECOMPRESS 984 985 /* ************************************************************************** 986 * Decompression 987 ***************************************************************************/ 988 typedef struct { 989 void* srcBuffer; 990 size_t srcBufferSize; 991 size_t srcBufferLoaded; 992 void* dstBuffer; 993 size_t dstBufferSize; 994 ZSTD_DStream* dctx; 995 FILE* dstFile; 996 } dRess_t; 997 998 static dRess_t FIO_createDResources(const char* dictFileName) 999 { 1000 dRess_t ress; 1001 memset(&ress, 0, sizeof(ress)); 1002 1003 /* Allocation */ 1004 ress.dctx = ZSTD_createDStream(); 1005 if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); 1006 CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) ); 1007 ress.srcBufferSize = ZSTD_DStreamInSize(); 1008 ress.srcBuffer = malloc(ress.srcBufferSize); 1009 ress.dstBufferSize = ZSTD_DStreamOutSize(); 1010 ress.dstBuffer = malloc(ress.dstBufferSize); 1011 if (!ress.srcBuffer || !ress.dstBuffer) 1012 EXM_THROW(61, "Allocation error : not enough memory"); 1013 1014 /* dictionary */ 1015 { void* dictBuffer; 1016 size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName); 1017 CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); 1018 free(dictBuffer); 1019 } 1020 1021 return ress; 1022 } 1023 1024 static void FIO_freeDResources(dRess_t ress) 1025 { 1026 CHECK( ZSTD_freeDStream(ress.dctx) ); 1027 free(ress.srcBuffer); 1028 free(ress.dstBuffer); 1029 } 1030 1031 1032 /** FIO_fwriteSparse() : 1033 * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ 1034 static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) 1035 { 1036 const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 1037 size_t bufferSizeT = bufferSize / sizeof(size_t); 1038 const size_t* const bufferTEnd = bufferT + bufferSizeT; 1039 const size_t* ptrT = bufferT; 1040 static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ 1041 1042 if (!g_sparseFileSupport) { /* normal write */ 1043 size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); 1044 if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); 1045 return 0; 1046 } 1047 1048 /* avoid int overflow */ 1049 if (storedSkips > 1 GB) { 1050 int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); 1051 if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); 1052 storedSkips -= 1 GB; 1053 } 1054 1055 while (ptrT < bufferTEnd) { 1056 size_t seg0SizeT = segmentSizeT; 1057 size_t nb0T; 1058 1059 /* count leading zeros */ 1060 if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 1061 bufferSizeT -= seg0SizeT; 1062 for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 1063 storedSkips += (unsigned)(nb0T * sizeof(size_t)); 1064 1065 if (nb0T != seg0SizeT) { /* not all 0s */ 1066 int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1067 if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); 1068 storedSkips = 0; 1069 seg0SizeT -= nb0T; 1070 ptrT += nb0T; 1071 { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); 1072 if (sizeCheck != seg0SizeT) 1073 EXM_THROW(73, "Write error : cannot write decoded block"); 1074 } } 1075 ptrT += seg0SizeT; 1076 } 1077 1078 { static size_t const maskT = sizeof(size_t)-1; 1079 if (bufferSize & maskT) { 1080 /* size not multiple of sizeof(size_t) : implies end of block */ 1081 const char* const restStart = (const char*)bufferTEnd; 1082 const char* restPtr = restStart; 1083 size_t restSize = bufferSize & maskT; 1084 const char* const restEnd = restStart + restSize; 1085 for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 1086 storedSkips += (unsigned) (restPtr - restStart); 1087 if (restPtr != restEnd) { 1088 int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1089 if (seekResult) 1090 EXM_THROW(74, "Sparse skip error ; try --no-sparse"); 1091 storedSkips = 0; 1092 { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); 1093 if (sizeCheck != (size_t)(restEnd - restPtr)) 1094 EXM_THROW(75, "Write error : cannot write decoded end of block"); 1095 } } } } 1096 1097 return storedSkips; 1098 } 1099 1100 static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) 1101 { 1102 if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ 1103 int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1104 if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)"); 1105 { const char lastZeroByte[1] = { 0 }; 1106 size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); 1107 if (sizeCheck != 1) 1108 EXM_THROW(69, "Write error : cannot write last zero"); 1109 } } 1110 } 1111 1112 1113 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode 1114 @return : 0 (no error) */ 1115 static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) 1116 { 1117 size_t const blockSize = MIN(64 KB, bufferSize); 1118 size_t readFromInput = 1; 1119 unsigned storedSkips = 0; 1120 1121 /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ 1122 { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); 1123 if (sizeCheck != alreadyLoaded) { 1124 DISPLAYLEVEL(1, "Pass-through write error \n"); 1125 return 1; 1126 } } 1127 1128 while (readFromInput) { 1129 readFromInput = fread(buffer, 1, blockSize, finput); 1130 storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips); 1131 } 1132 1133 FIO_fwriteSparseEnd(foutput, storedSkips); 1134 return 0; 1135 } 1136 1137 static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName) 1138 { 1139 ZSTD_frameHeader header; 1140 /* No special help for these errors */ 1141 if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) 1142 return; 1143 /* Try to decode the frame header */ 1144 ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); 1145 if (ret == 0) { 1146 U32 const windowSize = (U32)header.windowSize; 1147 U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0); 1148 U32 const windowMB = (windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0); 1149 assert(header.windowSize <= (U64)((U32)-1)); 1150 assert(g_memLimit > 0); 1151 DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n", 1152 srcFileName, header.windowSize, g_memLimit); 1153 if (windowLog <= ZSTD_WINDOWLOG_MAX) { 1154 DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n", 1155 srcFileName, windowLog, windowMB); 1156 return; 1157 } 1158 } else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) { 1159 DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n", 1160 srcFileName, ZSTD_getErrorName(ret)); 1161 return; 1162 } 1163 DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n", 1164 srcFileName, ZSTD_WINDOWLOG_MAX); 1165 } 1166 1167 /** FIO_decompressFrame() : 1168 * @return : size of decoded zstd frame, or an error code 1169 */ 1170 #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) 1171 unsigned long long FIO_decompressZstdFrame(dRess_t* ress, 1172 FILE* finput, 1173 const char* srcFileName, 1174 U64 alreadyDecoded) 1175 { 1176 U64 frameSize = 0; 1177 U32 storedSkips = 0; 1178 1179 size_t const srcFileLength = strlen(srcFileName); 1180 if (srcFileLength>20) srcFileName += srcFileLength-20; /* display last 20 characters only */ 1181 1182 ZSTD_resetDStream(ress->dctx); 1183 1184 /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ 1185 { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; 1186 if (ress->srcBufferLoaded < toDecode) { 1187 size_t const toRead = toDecode - ress->srcBufferLoaded; 1188 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1189 ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); 1190 } } 1191 1192 /* Main decompression Loop */ 1193 while (1) { 1194 ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; 1195 ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; 1196 size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); 1197 if (ZSTD_isError(readSizeHint)) { 1198 DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", 1199 srcFileName, ZSTD_getErrorName(readSizeHint)); 1200 FIO_zstdErrorHelp(ress, readSizeHint, srcFileName); 1201 return FIO_ERROR_FRAME_DECODING; 1202 } 1203 1204 /* Write block */ 1205 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); 1206 frameSize += outBuff.pos; 1207 DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", 1208 srcFileName, (U32)((alreadyDecoded+frameSize)>>20) ); 1209 1210 if (inBuff.pos > 0) { 1211 memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); 1212 ress->srcBufferLoaded -= inBuff.pos; 1213 } 1214 1215 if (readSizeHint == 0) break; /* end of frame */ 1216 if (inBuff.size != inBuff.pos) { 1217 DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n", 1218 srcFileName); 1219 return FIO_ERROR_FRAME_DECODING; 1220 } 1221 1222 /* Fill input buffer */ 1223 { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ 1224 if (ress->srcBufferLoaded < toDecode) { 1225 size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ 1226 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1227 size_t const readSize = fread(startPosition, 1, toRead, finput); 1228 if (readSize==0) { 1229 DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", 1230 srcFileName); 1231 return FIO_ERROR_FRAME_DECODING; 1232 } 1233 ress->srcBufferLoaded += readSize; 1234 } } } 1235 1236 FIO_fwriteSparseEnd(ress->dstFile, storedSkips); 1237 1238 return frameSize; 1239 } 1240 1241 1242 #ifdef ZSTD_GZDECOMPRESS 1243 static unsigned long long FIO_decompressGzFrame(dRess_t* ress, 1244 FILE* srcFile, const char* srcFileName) 1245 { 1246 unsigned long long outFileSize = 0; 1247 z_stream strm; 1248 int flush = Z_NO_FLUSH; 1249 int decodingError = 0; 1250 1251 strm.zalloc = Z_NULL; 1252 strm.zfree = Z_NULL; 1253 strm.opaque = Z_NULL; 1254 strm.next_in = 0; 1255 strm.avail_in = 0; 1256 /* see http://www.zlib.net/manual.html */ 1257 if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) 1258 return FIO_ERROR_FRAME_DECODING; 1259 1260 strm.next_out = (Bytef*)ress->dstBuffer; 1261 strm.avail_out = (uInt)ress->dstBufferSize; 1262 strm.avail_in = (uInt)ress->srcBufferLoaded; 1263 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1264 1265 for ( ; ; ) { 1266 int ret; 1267 if (strm.avail_in == 0) { 1268 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 1269 if (ress->srcBufferLoaded == 0) flush = Z_FINISH; 1270 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1271 strm.avail_in = (uInt)ress->srcBufferLoaded; 1272 } 1273 ret = inflate(&strm, flush); 1274 if (ret == Z_BUF_ERROR) { 1275 DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); 1276 decodingError = 1; break; 1277 } 1278 if (ret != Z_OK && ret != Z_STREAM_END) { 1279 DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); 1280 decodingError = 1; break; 1281 } 1282 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 1283 if (decompBytes) { 1284 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { 1285 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1286 decodingError = 1; break; 1287 } 1288 outFileSize += decompBytes; 1289 strm.next_out = (Bytef*)ress->dstBuffer; 1290 strm.avail_out = (uInt)ress->dstBufferSize; 1291 } 1292 } 1293 if (ret == Z_STREAM_END) break; 1294 } 1295 1296 if (strm.avail_in > 0) 1297 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 1298 ress->srcBufferLoaded = strm.avail_in; 1299 if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ 1300 && (decodingError==0) ) { 1301 DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); 1302 decodingError = 1; 1303 } 1304 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 1305 } 1306 #endif 1307 1308 1309 #ifdef ZSTD_LZMADECOMPRESS 1310 static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) 1311 { 1312 unsigned long long outFileSize = 0; 1313 lzma_stream strm = LZMA_STREAM_INIT; 1314 lzma_action action = LZMA_RUN; 1315 lzma_ret initRet; 1316 int decodingError = 0; 1317 1318 strm.next_in = 0; 1319 strm.avail_in = 0; 1320 if (plain_lzma) { 1321 initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ 1322 } else { 1323 initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ 1324 } 1325 1326 if (initRet != LZMA_OK) { 1327 DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", 1328 plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", 1329 srcFileName, initRet); 1330 return FIO_ERROR_FRAME_DECODING; 1331 } 1332 1333 strm.next_out = (BYTE*)ress->dstBuffer; 1334 strm.avail_out = ress->dstBufferSize; 1335 strm.next_in = (BYTE const*)ress->srcBuffer; 1336 strm.avail_in = ress->srcBufferLoaded; 1337 1338 for ( ; ; ) { 1339 lzma_ret ret; 1340 if (strm.avail_in == 0) { 1341 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 1342 if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; 1343 strm.next_in = (BYTE const*)ress->srcBuffer; 1344 strm.avail_in = ress->srcBufferLoaded; 1345 } 1346 ret = lzma_code(&strm, action); 1347 1348 if (ret == LZMA_BUF_ERROR) { 1349 DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); 1350 decodingError = 1; break; 1351 } 1352 if (ret != LZMA_OK && ret != LZMA_STREAM_END) { 1353 DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", 1354 srcFileName, ret); 1355 decodingError = 1; break; 1356 } 1357 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 1358 if (decompBytes) { 1359 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { 1360 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1361 decodingError = 1; break; 1362 } 1363 outFileSize += decompBytes; 1364 strm.next_out = (BYTE*)ress->dstBuffer; 1365 strm.avail_out = ress->dstBufferSize; 1366 } } 1367 if (ret == LZMA_STREAM_END) break; 1368 } 1369 1370 if (strm.avail_in > 0) 1371 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 1372 ress->srcBufferLoaded = strm.avail_in; 1373 lzma_end(&strm); 1374 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 1375 } 1376 #endif 1377 1378 #ifdef ZSTD_LZ4DECOMPRESS 1379 static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, 1380 FILE* srcFile, const char* srcFileName) 1381 { 1382 unsigned long long filesize = 0; 1383 LZ4F_errorCode_t nextToLoad; 1384 LZ4F_decompressionContext_t dCtx; 1385 LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 1386 int decodingError = 0; 1387 1388 if (LZ4F_isError(errorCode)) { 1389 DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); 1390 return FIO_ERROR_FRAME_DECODING; 1391 } 1392 1393 /* Init feed with magic number (already consumed from FILE* sFile) */ 1394 { size_t inSize = 4; 1395 size_t outSize= 0; 1396 MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); 1397 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); 1398 if (LZ4F_isError(nextToLoad)) { 1399 DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", 1400 srcFileName, LZ4F_getErrorName(nextToLoad)); 1401 LZ4F_freeDecompressionContext(dCtx); 1402 return FIO_ERROR_FRAME_DECODING; 1403 } } 1404 1405 /* Main Loop */ 1406 for (;nextToLoad;) { 1407 size_t readSize; 1408 size_t pos = 0; 1409 size_t decodedBytes = ress->dstBufferSize; 1410 1411 /* Read input */ 1412 if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; 1413 readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); 1414 if (!readSize) break; /* reached end of file or stream */ 1415 1416 while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ 1417 /* Decode Input (at least partially) */ 1418 size_t remaining = readSize - pos; 1419 decodedBytes = ress->dstBufferSize; 1420 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); 1421 if (LZ4F_isError(nextToLoad)) { 1422 DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", 1423 srcFileName, LZ4F_getErrorName(nextToLoad)); 1424 decodingError = 1; break; 1425 } 1426 pos += remaining; 1427 1428 /* Write Block */ 1429 if (decodedBytes) { 1430 if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { 1431 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1432 decodingError = 1; break; 1433 } 1434 filesize += decodedBytes; 1435 DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); 1436 } 1437 1438 if (!nextToLoad) break; 1439 } 1440 } 1441 /* can be out because readSize == 0, which could be an fread() error */ 1442 if (ferror(srcFile)) { 1443 DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); 1444 decodingError=1; 1445 } 1446 1447 if (nextToLoad!=0) { 1448 DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); 1449 decodingError=1; 1450 } 1451 1452 LZ4F_freeDecompressionContext(dCtx); 1453 ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ 1454 1455 return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; 1456 } 1457 #endif 1458 1459 1460 1461 /** FIO_decompressFrames() : 1462 * Find and decode frames inside srcFile 1463 * srcFile presumed opened and valid 1464 * @return : 0 : OK 1465 * 1 : error 1466 */ 1467 static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, 1468 const char* dstFileName, const char* srcFileName) 1469 { 1470 unsigned readSomething = 0; 1471 unsigned long long filesize = 0; 1472 assert(srcFile != NULL); 1473 1474 /* for each frame */ 1475 for ( ; ; ) { 1476 /* check magic number -> version */ 1477 size_t const toRead = 4; 1478 const BYTE* const buf = (const BYTE*)ress.srcBuffer; 1479 if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ 1480 ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, 1481 (size_t)1, toRead - ress.srcBufferLoaded, srcFile); 1482 if (ress.srcBufferLoaded==0) { 1483 if (readSomething==0) { /* srcFile is empty (which is invalid) */ 1484 DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); 1485 return 1; 1486 } /* else, just reached frame boundary */ 1487 break; /* no more input */ 1488 } 1489 readSomething = 1; /* there is at least 1 byte in srcFile */ 1490 if (ress.srcBufferLoaded < toRead) { 1491 DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); 1492 return 1; 1493 } 1494 if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { 1495 unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize); 1496 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1497 filesize += frameSize; 1498 } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ 1499 #ifdef ZSTD_GZDECOMPRESS 1500 unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); 1501 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1502 filesize += frameSize; 1503 #else 1504 DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); 1505 return 1; 1506 #endif 1507 } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ 1508 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ 1509 #ifdef ZSTD_LZMADECOMPRESS 1510 unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); 1511 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1512 filesize += frameSize; 1513 #else 1514 DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); 1515 return 1; 1516 #endif 1517 } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { 1518 #ifdef ZSTD_LZ4DECOMPRESS 1519 unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); 1520 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1521 filesize += frameSize; 1522 #else 1523 DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); 1524 return 1; 1525 #endif 1526 } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ 1527 return FIO_passThrough(ress.dstFile, srcFile, 1528 ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); 1529 } else { 1530 DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); 1531 return 1; 1532 } } /* for each frame */ 1533 1534 /* Final Status */ 1535 DISPLAYLEVEL(2, "\r%79s\r", ""); 1536 DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); 1537 1538 return 0; 1539 } 1540 1541 1542 /** FIO_decompressSrcFile() : 1543 Decompression `srcFileName` into `ress.dstFile` 1544 @return : 0 : OK 1545 1 : operation not started 1546 */ 1547 static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) 1548 { 1549 FILE* srcFile; 1550 int result; 1551 1552 if (UTIL_isDirectory(srcFileName)) { 1553 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 1554 return 1; 1555 } 1556 1557 srcFile = FIO_openSrcFile(srcFileName); 1558 if (srcFile==NULL) return 1; 1559 1560 result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName); 1561 1562 /* Close file */ 1563 if (fclose(srcFile)) { 1564 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ 1565 return 1; 1566 } 1567 if ( g_removeSrcFile /* --rm */ 1568 && (result==0) /* decompression successful */ 1569 && strcmp(srcFileName, stdinmark) ) /* not stdin */ { 1570 /* We must clear the handler, since after this point calling it would 1571 * delete both the source and destination files. 1572 */ 1573 clearHandler(); 1574 if (remove(srcFileName)) { 1575 /* failed to remove src file */ 1576 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 1577 return 1; 1578 } } 1579 return result; 1580 } 1581 1582 1583 /** FIO_decompressFile_extRess() : 1584 decompress `srcFileName` into `dstFileName` 1585 @return : 0 : OK 1586 1 : operation aborted (src not available, dst already taken, etc.) 1587 */ 1588 static int FIO_decompressDstFile(dRess_t ress, 1589 const char* dstFileName, const char* srcFileName) 1590 { 1591 int result; 1592 stat_t statbuf; 1593 int stat_result = 0; 1594 1595 ress.dstFile = FIO_openDstFile(dstFileName); 1596 if (ress.dstFile==0) return 1; 1597 /* Must ony be added after FIO_openDstFile() succeeds. 1598 * Otherwise we may delete the destination file if at already exists, and 1599 * the user presses Ctrl-C when asked if they wish to overwrite. 1600 */ 1601 addHandler(dstFileName); 1602 1603 if ( strcmp(srcFileName, stdinmark) 1604 && UTIL_getFileStat(srcFileName, &statbuf) ) 1605 stat_result = 1; 1606 result = FIO_decompressSrcFile(ress, dstFileName, srcFileName); 1607 clearHandler(); 1608 1609 if (fclose(ress.dstFile)) { 1610 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 1611 result = 1; 1612 } 1613 1614 if ( (result != 0) /* operation failure */ 1615 && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ 1616 && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */ 1617 remove(dstFileName); /* remove decompression artefact; note don't do anything special if remove() fails */ 1618 else { /* operation success */ 1619 if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */ 1620 && strcmp(dstFileName, nulmark) /* special case : don't chmod /dev/null */ 1621 && stat_result ) /* file permissions correctly extracted from src */ 1622 UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ 1623 } 1624 1625 signal(SIGINT, SIG_DFL); 1626 1627 return result; 1628 } 1629 1630 1631 int FIO_decompressFilename(const char* dstFileName, const char* srcFileName, 1632 const char* dictFileName) 1633 { 1634 dRess_t const ress = FIO_createDResources(dictFileName); 1635 1636 int const decodingError = FIO_decompressDstFile(ress, dstFileName, srcFileName); 1637 1638 FIO_freeDResources(ress); 1639 return decodingError; 1640 } 1641 1642 1643 #define MAXSUFFIXSIZE 8 1644 int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, 1645 const char* outFileName, 1646 const char* dictFileName) 1647 { 1648 int skippedFiles = 0; 1649 int missingFiles = 0; 1650 dRess_t ress = FIO_createDResources(dictFileName); 1651 1652 if (outFileName) { 1653 unsigned u; 1654 ress.dstFile = FIO_openDstFile(outFileName); 1655 if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); 1656 for (u=0; u<nbFiles; u++) 1657 missingFiles += FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]); 1658 if (fclose(ress.dstFile)) 1659 EXM_THROW(72, "Write error : cannot properly close output file"); 1660 } else { 1661 size_t suffixSize; 1662 size_t dfnSize = FNSPACE; 1663 unsigned u; 1664 char* dstFileName = (char*)malloc(FNSPACE); 1665 if (dstFileName==NULL) 1666 EXM_THROW(73, "not enough memory for dstFileName"); 1667 for (u=0; u<nbFiles; u++) { /* create dstFileName */ 1668 const char* const srcFileName = srcNamesTable[u]; 1669 const char* const suffixPtr = strrchr(srcFileName, '.'); 1670 size_t const sfnSize = strlen(srcFileName); 1671 if (!suffixPtr) { 1672 DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", 1673 srcFileName); 1674 skippedFiles++; 1675 continue; 1676 } 1677 suffixSize = strlen(suffixPtr); 1678 if (dfnSize+suffixSize <= sfnSize+1) { 1679 free(dstFileName); 1680 dfnSize = sfnSize + 20; 1681 dstFileName = (char*)malloc(dfnSize); 1682 if (dstFileName==NULL) 1683 EXM_THROW(74, "not enough memory for dstFileName"); 1684 } 1685 if (sfnSize <= suffixSize 1686 || (strcmp(suffixPtr, GZ_EXTENSION) 1687 && strcmp(suffixPtr, XZ_EXTENSION) 1688 && strcmp(suffixPtr, ZSTD_EXTENSION) 1689 && strcmp(suffixPtr, LZMA_EXTENSION) 1690 && strcmp(suffixPtr, LZ4_EXTENSION)) ) { 1691 DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s/%s expected) -- ignored \n", 1692 srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION, LZ4_EXTENSION); 1693 skippedFiles++; 1694 continue; 1695 } else { 1696 memcpy(dstFileName, srcFileName, sfnSize - suffixSize); 1697 dstFileName[sfnSize-suffixSize] = '\0'; 1698 } 1699 missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName); 1700 } 1701 free(dstFileName); 1702 } 1703 1704 FIO_freeDResources(ress); 1705 return missingFiles + skippedFiles; 1706 } 1707 1708 1709 1710 /* ************************************************************************** 1711 * .zst file info (--list command) 1712 ***************************************************************************/ 1713 1714 typedef struct { 1715 U64 decompressedSize; 1716 U64 compressedSize; 1717 U64 windowSize; 1718 int numActualFrames; 1719 int numSkippableFrames; 1720 int decompUnavailable; 1721 int usesCheck; 1722 U32 nbFiles; 1723 } fileInfo_t; 1724 1725 /** getFileInfo() : 1726 * Reads information from file, stores in *info 1727 * @return : 0 if successful 1728 * 1 for frame analysis error 1729 * 2 for file not compressed with zstd 1730 * 3 for cases in which file could not be opened. 1731 */ 1732 static int getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName){ 1733 int detectError = 0; 1734 FILE* const srcFile = FIO_openSrcFile(inFileName); 1735 if (srcFile == NULL) { 1736 DISPLAY("Error: could not open source file %s\n", inFileName); 1737 return 3; 1738 } 1739 info->compressedSize = UTIL_getFileSize(inFileName); 1740 1741 /* begin analyzing frame */ 1742 for ( ; ; ) { 1743 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; 1744 size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); 1745 if (numBytesRead < ZSTD_frameHeaderSize_min) { 1746 if ( feof(srcFile) 1747 && (numBytesRead == 0) 1748 && (info->compressedSize > 0) 1749 && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { 1750 break; 1751 } 1752 else if (feof(srcFile)) { 1753 DISPLAY("Error: reached end of file with incomplete frame\n"); 1754 detectError = 2; 1755 break; 1756 } 1757 else { 1758 DISPLAY("Error: did not reach end of file but ran out of frames\n"); 1759 detectError = 1; 1760 break; 1761 } 1762 } 1763 { U32 const magicNumber = MEM_readLE32(headerBuffer); 1764 /* Zstandard frame */ 1765 if (magicNumber == ZSTD_MAGICNUMBER) { 1766 ZSTD_frameHeader header; 1767 U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); 1768 if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { 1769 info->decompUnavailable = 1; 1770 } else { 1771 info->decompressedSize += frameContentSize; 1772 } 1773 if (ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0) { 1774 DISPLAY("Error: could not decode frame header\n"); 1775 detectError = 1; 1776 break; 1777 } 1778 info->windowSize = header.windowSize; 1779 /* move to the end of the frame header */ 1780 { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); 1781 if (ZSTD_isError(headerSize)) { 1782 DISPLAY("Error: could not determine frame header size\n"); 1783 detectError = 1; 1784 break; 1785 } 1786 { int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); 1787 if (ret != 0) { 1788 DISPLAY("Error: could not move to end of frame header\n"); 1789 detectError = 1; 1790 break; 1791 } } } 1792 1793 /* skip the rest of the blocks in the frame */ 1794 { int lastBlock = 0; 1795 do { 1796 BYTE blockHeaderBuffer[3]; 1797 size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); 1798 if (readBytes != 3) { 1799 DISPLAY("There was a problem reading the block header\n"); 1800 detectError = 1; 1801 break; 1802 } 1803 { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); 1804 U32 const blockTypeID = (blockHeader >> 1) & 3; 1805 U32 const isRLE = (blockTypeID == 1); 1806 U32 const isWrongBlock = (blockTypeID == 3); 1807 long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); 1808 if (isWrongBlock) { 1809 DISPLAY("Error: unsupported block type \n"); 1810 detectError = 1; 1811 break; 1812 } 1813 lastBlock = blockHeader & 1; 1814 { int const ret = fseek(srcFile, blockSize, SEEK_CUR); 1815 if (ret != 0) { 1816 DISPLAY("Error: could not skip to end of block\n"); 1817 detectError = 1; 1818 break; 1819 } } } 1820 } while (lastBlock != 1); 1821 1822 if (detectError) break; 1823 } 1824 1825 /* check if checksum is used */ 1826 { BYTE const frameHeaderDescriptor = headerBuffer[4]; 1827 int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; 1828 if (contentChecksumFlag) { 1829 int const ret = fseek(srcFile, 4, SEEK_CUR); 1830 info->usesCheck = 1; 1831 if (ret != 0) { 1832 DISPLAY("Error: could not skip past checksum\n"); 1833 detectError = 1; 1834 break; 1835 } } } 1836 info->numActualFrames++; 1837 } 1838 /* Skippable frame */ 1839 else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { 1840 U32 const frameSize = MEM_readLE32(headerBuffer + 4); 1841 long const seek = (long)(8 + frameSize - numBytesRead); 1842 int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR); 1843 if (ret != 0) { 1844 DISPLAY("Error: could not find end of skippable frame\n"); 1845 detectError = 1; 1846 break; 1847 } 1848 info->numSkippableFrames++; 1849 } 1850 /* unknown content */ 1851 else { 1852 detectError = 2; 1853 break; 1854 } 1855 } 1856 } /* end analyzing frame */ 1857 fclose(srcFile); 1858 info->nbFiles = 1; 1859 return detectError; 1860 } 1861 1862 static int getFileInfo(fileInfo_t* info, const char* srcFileName) 1863 { 1864 int const isAFile = UTIL_isRegularFile(srcFileName); 1865 if (!isAFile) { 1866 DISPLAY("Error : %s is not a file", srcFileName); 1867 return 3; 1868 } 1869 return getFileInfo_fileConfirmed(info, srcFileName); 1870 } 1871 1872 1873 static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){ 1874 unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); 1875 const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; 1876 double const windowSizeUnit = (double)info->windowSize / unit; 1877 double const compressedSizeUnit = (double)info->compressedSize / unit; 1878 double const decompressedSizeUnit = (double)info->decompressedSize / unit; 1879 double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; 1880 const char* const checkString = (info->usesCheck ? "XXH64" : "None"); 1881 if (displayLevel <= 2) { 1882 if (!info->decompUnavailable) { 1883 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", 1884 info->numSkippableFrames + info->numActualFrames, 1885 info->numSkippableFrames, 1886 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 1887 ratio, checkString, inFileName); 1888 } else { 1889 DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n", 1890 info->numSkippableFrames + info->numActualFrames, 1891 info->numSkippableFrames, 1892 compressedSizeUnit, unitStr, 1893 checkString, inFileName); 1894 } 1895 } else { 1896 DISPLAYOUT("%s \n", inFileName); 1897 DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); 1898 if (info->numSkippableFrames) 1899 DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); 1900 DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", 1901 windowSizeUnit, unitStr, 1902 (unsigned long long)info->windowSize); 1903 DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", 1904 compressedSizeUnit, unitStr, 1905 (unsigned long long)info->compressedSize); 1906 if (!info->decompUnavailable) { 1907 DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", 1908 decompressedSizeUnit, unitStr, 1909 (unsigned long long)info->decompressedSize); 1910 DISPLAYOUT("Ratio: %.4f\n", ratio); 1911 } 1912 DISPLAYOUT("Check: %s\n", checkString); 1913 DISPLAYOUT("\n"); 1914 } 1915 } 1916 1917 static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) 1918 { 1919 fileInfo_t total; 1920 total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; 1921 total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; 1922 total.compressedSize = fi1.compressedSize + fi2.compressedSize; 1923 total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; 1924 total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; 1925 total.usesCheck = fi1.usesCheck & fi2.usesCheck; 1926 total.nbFiles = fi1.nbFiles + fi2.nbFiles; 1927 return total; 1928 } 1929 1930 static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){ 1931 fileInfo_t info; 1932 memset(&info, 0, sizeof(info)); 1933 { int const error = getFileInfo(&info, inFileName); 1934 if (error == 1) { 1935 /* display error, but provide output */ 1936 DISPLAY("An error occurred while getting file info \n"); 1937 } 1938 else if (error == 2) { 1939 DISPLAYOUT("File %s not compressed by zstd \n", inFileName); 1940 if (displayLevel > 2) DISPLAYOUT("\n"); 1941 return 1; 1942 } 1943 else if (error == 3) { 1944 /* error occurred while opening the file */ 1945 if (displayLevel > 2) DISPLAYOUT("\n"); 1946 return 1; 1947 } 1948 displayInfo(inFileName, &info, displayLevel); 1949 *total = FIO_addFInfo(*total, info); 1950 return error; 1951 } 1952 } 1953 1954 int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ 1955 if (numFiles == 0) { 1956 DISPLAYOUT("No files given\n"); 1957 return 0; 1958 } 1959 if (displayLevel <= 2) { 1960 DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); 1961 } 1962 { int error = 0; 1963 unsigned u; 1964 fileInfo_t total; 1965 memset(&total, 0, sizeof(total)); 1966 total.usesCheck = 1; 1967 for (u=0; u<numFiles;u++) { 1968 error |= FIO_listFile(&total, filenameTable[u], displayLevel); 1969 } 1970 if (numFiles > 1 && displayLevel <= 2) { /* display total */ 1971 unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); 1972 const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; 1973 double const compressedSizeUnit = (double)total.compressedSize / unit; 1974 double const decompressedSizeUnit = (double)total.decompressedSize / unit; 1975 double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize; 1976 const char* const checkString = (total.usesCheck ? "XXH64" : ""); 1977 DISPLAYOUT("----------------------------------------------------------------- \n"); 1978 if (total.decompUnavailable) { 1979 DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n", 1980 total.numSkippableFrames + total.numActualFrames, 1981 total.numSkippableFrames, 1982 compressedSizeUnit, unitStr, 1983 checkString, total.nbFiles); 1984 } else { 1985 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n", 1986 total.numSkippableFrames + total.numActualFrames, 1987 total.numSkippableFrames, 1988 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 1989 ratio, checkString, total.nbFiles); 1990 } } 1991 return error; 1992 } 1993 } 1994 1995 1996 #endif /* #ifndef ZSTD_NODECOMPRESS */ 1997