1 /* 2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /* ************************************* 13 * Compiler Options 14 ***************************************/ 15 #ifdef _MSC_VER /* Visual */ 16 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 17 # pragma warning(disable : 4204) /* non-constant aggregate initializer */ 18 #endif 19 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) 20 # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ 21 #endif 22 23 24 /*-************************************* 25 * Includes 26 ***************************************/ 27 #include "platform.h" /* Large Files support, SET_BINARY_MODE */ 28 #include "util.h" /* UTIL_getFileSize */ 29 #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ 30 #include <stdlib.h> /* malloc, free */ 31 #include <string.h> /* strcmp, strlen */ 32 #include <time.h> /* clock */ 33 #include <errno.h> /* errno */ 34 35 #if defined (_MSC_VER) 36 # include <sys/stat.h> 37 # include <io.h> 38 #endif 39 40 #include "bitstream.h" 41 #include "mem.h" 42 #include "fileio.h" 43 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ 44 #include "zstd.h" 45 #ifdef ZSTD_MULTITHREAD 46 # include "zstdmt_compress.h" 47 #endif 48 #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) 49 # include <zlib.h> 50 # if !defined(z_const) 51 # define z_const 52 # endif 53 #endif 54 #if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) 55 # include <lzma.h> 56 #endif 57 58 #define LZ4_MAGICNUMBER 0x184D2204 59 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) 60 # define LZ4F_ENABLE_OBSOLETE_ENUMS 61 # include <lz4frame.h> 62 # include <lz4.h> 63 #endif 64 65 66 /*-************************************* 67 * Constants 68 ***************************************/ 69 #define KB *(1<<10) 70 #define MB *(1<<20) 71 #define GB *(1U<<30) 72 73 #define _1BIT 0x01 74 #define _2BITS 0x03 75 #define _3BITS 0x07 76 #define _4BITS 0x0F 77 #define _6BITS 0x3F 78 #define _8BITS 0xFF 79 80 #define BLOCKSIZE (128 KB) 81 #define ROLLBUFFERSIZE (BLOCKSIZE*8*64) 82 83 #define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */ 84 85 #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ 86 87 #define FNSPACE 30 88 89 90 /*-************************************* 91 * Macros 92 ***************************************/ 93 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 94 #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) 95 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } 96 static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ 97 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } 98 99 #define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \ 100 if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ 101 { g_time = clock(); DISPLAY(__VA_ARGS__); \ 102 if (g_displayLevel>=4) fflush(stderr); } } } 103 static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; 104 static clock_t g_time = 0; 105 106 #undef MIN /* in case it would be already defined */ 107 #define MIN(a,b) ((a) < (b) ? (a) : (b)) 108 109 110 /*-************************************* 111 * Debug 112 ***************************************/ 113 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) 114 # include <assert.h> 115 #else 116 # ifndef assert 117 # define assert(condition) ((void)0) 118 # endif 119 #endif 120 121 #ifndef ZSTD_DEBUG 122 # define ZSTD_DEBUG 0 123 #endif 124 #define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__); 125 #define EXM_THROW(error, ...) \ 126 { \ 127 DISPLAYLEVEL(1, "zstd: "); \ 128 DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 129 DISPLAYLEVEL(1, "error %i : ", error); \ 130 DISPLAYLEVEL(1, __VA_ARGS__); \ 131 DISPLAYLEVEL(1, " \n"); \ 132 exit(error); \ 133 } 134 135 #define CHECK(f) { \ 136 size_t const err = f; \ 137 if (ZSTD_isError(err)) { \ 138 DEBUGLOG(1, "%s \n", #f); \ 139 EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \ 140 } } 141 142 143 /*-************************************ 144 * Signal (Ctrl-C trapping) 145 **************************************/ 146 #include <signal.h> 147 148 static const char* g_artefact = NULL; 149 static void INThandler(int sig) 150 { 151 assert(sig==SIGINT); (void)sig; 152 #if !defined(_MSC_VER) 153 signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ 154 #endif 155 if (g_artefact) remove(g_artefact); 156 DISPLAY("\n"); 157 exit(2); 158 } 159 160 161 /* ************************************************************ 162 * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW 163 ***************************************************************/ 164 #if defined(_MSC_VER) && _MSC_VER >= 1400 165 # define LONG_SEEK _fseeki64 166 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 167 # define LONG_SEEK fseeko 168 #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) 169 # define LONG_SEEK fseeko64 170 #elif defined(_WIN32) && !defined(__DJGPP__) 171 # include <windows.h> 172 static int LONG_SEEK(FILE* file, __int64 offset, int origin) { 173 LARGE_INTEGER off; 174 DWORD method; 175 off.QuadPart = offset; 176 if (origin == SEEK_END) 177 method = FILE_END; 178 else if (origin == SEEK_CUR) 179 method = FILE_CURRENT; 180 else 181 method = FILE_BEGIN; 182 183 if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) 184 return 0; 185 else 186 return -1; 187 } 188 #else 189 # define LONG_SEEK fseek 190 #endif 191 192 193 /*-************************************* 194 * Local Parameters - Not thread safe 195 ***************************************/ 196 static FIO_compressionType_t g_compressionType = FIO_zstdCompression; 197 void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } 198 static U32 g_overwrite = 0; 199 void FIO_overwriteMode(void) { g_overwrite=1; } 200 static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ 201 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } 202 static U32 g_dictIDFlag = 1; 203 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } 204 static U32 g_checksumFlag = 1; 205 void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } 206 static U32 g_removeSrcFile = 0; 207 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } 208 static U32 g_memLimit = 0; 209 void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } 210 static U32 g_nbThreads = 1; 211 void FIO_setNbThreads(unsigned nbThreads) { 212 #ifndef ZSTD_MULTITHREAD 213 if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); 214 #endif 215 g_nbThreads = nbThreads; 216 } 217 static U32 g_blockSize = 0; 218 void FIO_setBlockSize(unsigned blockSize) { 219 if (blockSize && g_nbThreads==1) 220 DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); 221 #ifdef ZSTD_MULTITHREAD 222 if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1) /* intentional underflow */ 223 DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10)); 224 #endif 225 g_blockSize = blockSize; 226 } 227 #define FIO_OVERLAP_LOG_NOTSET 9999 228 static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET; 229 void FIO_setOverlapLog(unsigned overlapLog){ 230 if (overlapLog && g_nbThreads==1) 231 DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); 232 g_overlapLog = overlapLog; 233 } 234 static U32 g_ldmFlag = 0; 235 void FIO_setLdmFlag(unsigned ldmFlag) { 236 g_ldmFlag = (ldmFlag>0); 237 } 238 static U32 g_ldmHashLog = 0; 239 void FIO_setLdmHashLog(unsigned ldmHashLog) { 240 g_ldmHashLog = ldmHashLog; 241 } 242 static U32 g_ldmMinMatch = 0; 243 void FIO_setLdmMinMatch(unsigned ldmMinMatch) { 244 g_ldmMinMatch = ldmMinMatch; 245 } 246 247 #define FIO_LDM_PARAM_NOTSET 9999 248 static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; 249 void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) { 250 g_ldmBucketSizeLog = ldmBucketSizeLog; 251 } 252 253 static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET; 254 void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) { 255 g_ldmHashEveryLog = ldmHashEveryLog; 256 } 257 258 259 260 /*-************************************* 261 * Functions 262 ***************************************/ 263 /** FIO_remove() : 264 * @result : Unlink `fileName`, even if it's read-only */ 265 static int FIO_remove(const char* path) 266 { 267 #if defined(_WIN32) || defined(WIN32) 268 /* windows doesn't allow remove read-only files, 269 * so try to make it writable first */ 270 chmod(path, _S_IWRITE); 271 #endif 272 return remove(path); 273 } 274 275 /** FIO_openSrcFile() : 276 * condition : `dstFileName` must be non-NULL. 277 * @result : FILE* to `dstFileName`, or NULL if it fails */ 278 static FILE* FIO_openSrcFile(const char* srcFileName) 279 { 280 FILE* f; 281 282 if (!strcmp (srcFileName, stdinmark)) { 283 DISPLAYLEVEL(4,"Using stdin for input\n"); 284 f = stdin; 285 SET_BINARY_MODE(stdin); 286 } else { 287 if (!UTIL_isRegularFile(srcFileName)) { 288 DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", 289 srcFileName); 290 return NULL; 291 } 292 f = fopen(srcFileName, "rb"); 293 if ( f==NULL ) 294 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 295 } 296 297 return f; 298 } 299 300 /** FIO_openDstFile() : 301 * condition : `dstFileName` must be non-NULL. 302 * @result : FILE* to `dstFileName`, or NULL if it fails */ 303 static FILE* FIO_openDstFile(const char* dstFileName) 304 { 305 FILE* f; 306 307 if (!strcmp (dstFileName, stdoutmark)) { 308 DISPLAYLEVEL(4,"Using stdout for output\n"); 309 f = stdout; 310 SET_BINARY_MODE(stdout); 311 if (g_sparseFileSupport==1) { 312 g_sparseFileSupport = 0; 313 DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 314 } 315 } else { 316 if (g_sparseFileSupport == 1) { 317 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; 318 } 319 if (strcmp (dstFileName, nulmark)) { 320 /* Check if destination file already exists */ 321 f = fopen( dstFileName, "rb" ); 322 if (f != 0) { /* dst file exists, prompt for overwrite authorization */ 323 fclose(f); 324 if (!g_overwrite) { 325 if (g_displayLevel <= 1) { 326 /* No interaction possible */ 327 DISPLAY("zstd: %s already exists; not overwritten \n", 328 dstFileName); 329 return NULL; 330 } 331 DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", 332 dstFileName); 333 { int ch = getchar(); 334 if ((ch!='Y') && (ch!='y')) { 335 DISPLAY(" not overwritten \n"); 336 return NULL; 337 } 338 /* flush rest of input line */ 339 while ((ch!=EOF) && (ch!='\n')) ch = getchar(); 340 } } 341 /* need to unlink */ 342 FIO_remove(dstFileName); 343 } } 344 f = fopen( dstFileName, "wb" ); 345 if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); 346 } 347 348 return f; 349 } 350 351 352 /*! FIO_createDictBuffer() : 353 * creates a buffer, pointed by `*bufferPtr`, 354 * loads `filename` content into it, up to DICTSIZE_MAX bytes. 355 * @return : loaded size 356 * if fileName==NULL, returns 0 and a NULL pointer 357 */ 358 static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) 359 { 360 FILE* fileHandle; 361 U64 fileSize; 362 363 *bufferPtr = NULL; 364 if (fileName == NULL) return 0; 365 366 DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); 367 fileHandle = fopen(fileName, "rb"); 368 if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); 369 fileSize = UTIL_getFileSize(fileName); 370 if (fileSize > DICTSIZE_MAX) { 371 EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", 372 fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */ 373 } 374 *bufferPtr = malloc((size_t)fileSize); 375 if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); 376 { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); 377 if (readSize!=fileSize) 378 EXM_THROW(35, "Error reading dictionary file %s", fileName); 379 } 380 fclose(fileHandle); 381 return (size_t)fileSize; 382 } 383 384 #ifndef ZSTD_NOCOMPRESS 385 386 /*-********************************************************************** 387 * Compression 388 ************************************************************************/ 389 typedef struct { 390 FILE* srcFile; 391 FILE* dstFile; 392 void* srcBuffer; 393 size_t srcBufferSize; 394 void* dstBuffer; 395 size_t dstBufferSize; 396 #if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD) 397 ZSTDMT_CCtx* cctx; 398 #else 399 ZSTD_CStream* cctx; 400 #endif 401 } cRess_t; 402 403 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, 404 U64 srcSize, 405 ZSTD_compressionParameters* comprParams) { 406 cRess_t ress; 407 memset(&ress, 0, sizeof(ress)); 408 409 #ifdef ZSTD_NEWAPI 410 ress.cctx = ZSTD_createCCtx(); 411 if (ress.cctx == NULL) 412 EXM_THROW(30, "allocation error : can't create ZSTD_CCtx"); 413 #elif defined(ZSTD_MULTITHREAD) 414 ress.cctx = ZSTDMT_createCCtx(g_nbThreads); 415 if (ress.cctx == NULL) 416 EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx"); 417 if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET)) 418 /* use complete window for overlap */ 419 ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9); 420 if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) 421 ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog); 422 #else 423 ress.cctx = ZSTD_createCStream(); 424 if (ress.cctx == NULL) 425 EXM_THROW(30, "allocation error : can't create ZSTD_CStream"); 426 #endif 427 ress.srcBufferSize = ZSTD_CStreamInSize(); 428 ress.srcBuffer = malloc(ress.srcBufferSize); 429 ress.dstBufferSize = ZSTD_CStreamOutSize(); 430 ress.dstBuffer = malloc(ress.dstBufferSize); 431 if (!ress.srcBuffer || !ress.dstBuffer) 432 EXM_THROW(31, "allocation error : not enough memory"); 433 434 /* dictionary */ 435 { void* dictBuffer; 436 size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ 437 if (dictFileName && (dictBuffer==NULL)) 438 EXM_THROW(32, "allocation error : can't create dictBuffer"); 439 440 #ifdef ZSTD_NEWAPI 441 { /* frame parameters */ 442 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); 443 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); 444 (void)srcSize; 445 /* compression level */ 446 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); 447 /* long distance matching */ 448 CHECK( ZSTD_CCtx_setParameter( 449 ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) ); 450 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) ); 451 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) ); 452 if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { 453 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) ); 454 } 455 if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) { 456 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) ); 457 } 458 /* compression parameters */ 459 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) ); 460 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) ); 461 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) ); 462 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) ); 463 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) ); 464 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); 465 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); 466 /* multi-threading */ 467 DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); 468 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); 469 /* dictionary */ 470 CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); 471 } 472 #elif defined(ZSTD_MULTITHREAD) 473 { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); 474 params.fParams.checksumFlag = g_checksumFlag; 475 params.fParams.noDictIDFlag = !g_dictIDFlag; 476 if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; 477 if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; 478 if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; 479 if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; 480 if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; 481 if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; 482 if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; 483 CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) ); 484 ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize); 485 } 486 #else 487 { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); 488 params.fParams.checksumFlag = g_checksumFlag; 489 params.fParams.noDictIDFlag = !g_dictIDFlag; 490 if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; 491 if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; 492 if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; 493 if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; 494 if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; 495 if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; 496 if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; 497 CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) ); 498 } 499 #endif 500 free(dictBuffer); 501 } 502 503 return ress; 504 } 505 506 static void FIO_freeCResources(cRess_t ress) 507 { 508 free(ress.srcBuffer); 509 free(ress.dstBuffer); 510 #if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD) 511 ZSTDMT_freeCCtx(ress.cctx); 512 #else 513 ZSTD_freeCStream(ress.cctx); /* never fails */ 514 #endif 515 } 516 517 518 #ifdef ZSTD_GZCOMPRESS 519 static unsigned long long FIO_compressGzFrame(cRess_t* ress, 520 const char* srcFileName, U64 const srcFileSize, 521 int compressionLevel, U64* readsize) 522 { 523 unsigned long long inFileSize = 0, outFileSize = 0; 524 z_stream strm; 525 int ret; 526 527 if (compressionLevel > Z_BEST_COMPRESSION) 528 compressionLevel = Z_BEST_COMPRESSION; 529 530 strm.zalloc = Z_NULL; 531 strm.zfree = Z_NULL; 532 strm.opaque = Z_NULL; 533 534 ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 535 15 /* maxWindowLogSize */ + 16 /* gzip only */, 536 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 537 if (ret != Z_OK) 538 EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); 539 540 strm.next_in = 0; 541 strm.avail_in = 0; 542 strm.next_out = (Bytef*)ress->dstBuffer; 543 strm.avail_out = (uInt)ress->dstBufferSize; 544 545 while (1) { 546 if (strm.avail_in == 0) { 547 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 548 if (inSize == 0) break; 549 inFileSize += inSize; 550 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 551 strm.avail_in = (uInt)inSize; 552 } 553 ret = deflate(&strm, Z_NO_FLUSH); 554 if (ret != Z_OK) 555 EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); 556 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 557 if (decompBytes) { 558 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) 559 EXM_THROW(73, "Write error : cannot write to output file"); 560 outFileSize += decompBytes; 561 strm.next_out = (Bytef*)ress->dstBuffer; 562 strm.avail_out = (uInt)ress->dstBufferSize; 563 } 564 } 565 if (!srcFileSize) 566 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 567 (U32)(inFileSize>>20), 568 (double)outFileSize/inFileSize*100) 569 else 570 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 571 (U32)(inFileSize>>20), (U32)(srcFileSize>>20), 572 (double)outFileSize/inFileSize*100); 573 } 574 575 while (1) { 576 ret = deflate(&strm, Z_FINISH); 577 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 578 if (decompBytes) { 579 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) 580 EXM_THROW(75, "Write error : cannot write to output file"); 581 outFileSize += decompBytes; 582 strm.next_out = (Bytef*)ress->dstBuffer; 583 strm.avail_out = (uInt)ress->dstBufferSize; 584 } } 585 if (ret == Z_STREAM_END) break; 586 if (ret != Z_BUF_ERROR) 587 EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); 588 } 589 590 ret = deflateEnd(&strm); 591 if (ret != Z_OK) 592 EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); 593 *readsize = inFileSize; 594 595 return outFileSize; 596 } 597 #endif 598 599 600 #ifdef ZSTD_LZMACOMPRESS 601 static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, 602 const char* srcFileName, U64 const srcFileSize, 603 int compressionLevel, U64* readsize, int plain_lzma) 604 { 605 unsigned long long inFileSize = 0, outFileSize = 0; 606 lzma_stream strm = LZMA_STREAM_INIT; 607 lzma_action action = LZMA_RUN; 608 lzma_ret ret; 609 610 if (compressionLevel < 0) compressionLevel = 0; 611 if (compressionLevel > 9) compressionLevel = 9; 612 613 if (plain_lzma) { 614 lzma_options_lzma opt_lzma; 615 if (lzma_lzma_preset(&opt_lzma, compressionLevel)) 616 EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); 617 ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ 618 if (ret != LZMA_OK) 619 EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); 620 } else { 621 ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ 622 if (ret != LZMA_OK) 623 EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); 624 } 625 626 strm.next_in = 0; 627 strm.avail_in = 0; 628 strm.next_out = (BYTE*)ress->dstBuffer; 629 strm.avail_out = ress->dstBufferSize; 630 631 while (1) { 632 if (strm.avail_in == 0) { 633 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 634 if (inSize == 0) action = LZMA_FINISH; 635 inFileSize += inSize; 636 strm.next_in = (BYTE const*)ress->srcBuffer; 637 strm.avail_in = inSize; 638 } 639 640 ret = lzma_code(&strm, action); 641 642 if (ret != LZMA_OK && ret != LZMA_STREAM_END) 643 EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); 644 { size_t const compBytes = ress->dstBufferSize - strm.avail_out; 645 if (compBytes) { 646 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) 647 EXM_THROW(73, "Write error : cannot write to output file"); 648 outFileSize += compBytes; 649 strm.next_out = (BYTE*)ress->dstBuffer; 650 strm.avail_out = ress->dstBufferSize; 651 } } 652 if (!srcFileSize) 653 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 654 (U32)(inFileSize>>20), 655 (double)outFileSize/inFileSize*100) 656 else 657 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 658 (U32)(inFileSize>>20), (U32)(srcFileSize>>20), 659 (double)outFileSize/inFileSize*100); 660 if (ret == LZMA_STREAM_END) break; 661 } 662 663 lzma_end(&strm); 664 *readsize = inFileSize; 665 666 return outFileSize; 667 } 668 #endif 669 670 #ifdef ZSTD_LZ4COMPRESS 671 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 672 static unsigned long long FIO_compressLz4Frame(cRess_t* ress, 673 const char* srcFileName, U64 const srcFileSize, 674 int compressionLevel, U64* readsize) 675 { 676 unsigned long long inFileSize = 0, outFileSize = 0; 677 678 LZ4F_preferences_t prefs; 679 LZ4F_compressionContext_t ctx; 680 681 LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 682 if (LZ4F_isError(errorCode)) 683 EXM_THROW(31, "zstd: failed to create lz4 compression context"); 684 685 memset(&prefs, 0, sizeof(prefs)); 686 687 #if LZ4_VERSION_NUMBER <= 10600 688 #define LZ4F_blockIndependent blockIndependent 689 #define LZ4F_max4MB max4MB 690 #endif 691 692 prefs.autoFlush = 1; 693 prefs.compressionLevel = compressionLevel; 694 prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* stick to defaults for lz4 cli */ 695 prefs.frameInfo.blockSizeID = LZ4F_max4MB; 696 prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; 697 #if LZ4_VERSION_NUMBER >= 10600 698 prefs.frameInfo.contentSize = srcFileSize; 699 #endif 700 701 { 702 size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max4MB); 703 size_t readSize; 704 size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); 705 if (LZ4F_isError(headerSize)) 706 EXM_THROW(33, "File header generation failed : %s", 707 LZ4F_getErrorName(headerSize)); 708 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); 709 if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } 710 outFileSize += headerSize; 711 712 /* Read first block */ 713 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 714 inFileSize += readSize; 715 716 /* Main Loop */ 717 while (readSize>0) { 718 size_t outSize; 719 720 /* Compress Block */ 721 outSize = LZ4F_compressUpdate(ctx, ress->dstBuffer, ress->dstBufferSize, ress->srcBuffer, readSize, NULL); 722 if (LZ4F_isError(outSize)) 723 EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", 724 srcFileName, LZ4F_getErrorName(outSize)); 725 outFileSize += outSize; 726 if (!srcFileSize) 727 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 728 (U32)(inFileSize>>20), 729 (double)outFileSize/inFileSize*100) 730 else 731 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 732 (U32)(inFileSize>>20), (U32)(srcFileSize>>20), 733 (double)outFileSize/inFileSize*100); 734 735 /* Write Block */ 736 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); 737 if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } 738 739 /* Read next block */ 740 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 741 inFileSize += readSize; 742 } 743 if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); 744 745 /* End of Stream mark */ 746 headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); 747 if (LZ4F_isError(headerSize)) 748 EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", 749 srcFileName, LZ4F_getErrorName(headerSize)); 750 751 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); 752 if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } 753 outFileSize += headerSize; 754 } 755 756 *readsize = inFileSize; 757 LZ4F_freeCompressionContext(ctx); 758 759 return outFileSize; 760 } 761 #endif 762 763 764 /*! FIO_compressFilename_internal() : 765 * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. 766 * @return : 0 : compression completed correctly, 767 * 1 : missing or pb opening srcFileName 768 */ 769 static int FIO_compressFilename_internal(cRess_t ress, 770 const char* dstFileName, const char* srcFileName, int compressionLevel) 771 { 772 FILE* const srcFile = ress.srcFile; 773 FILE* const dstFile = ress.dstFile; 774 U64 readsize = 0; 775 U64 compressedfilesize = 0; 776 U64 const fileSize = UTIL_getFileSize(srcFileName); 777 DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize); 778 779 switch (g_compressionType) { 780 case FIO_zstdCompression: 781 break; 782 783 case FIO_gzipCompression: 784 #ifdef ZSTD_GZCOMPRESS 785 compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 786 #else 787 (void)compressionLevel; 788 EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", 789 srcFileName); 790 #endif 791 goto finish; 792 793 case FIO_xzCompression: 794 case FIO_lzmaCompression: 795 #ifdef ZSTD_LZMACOMPRESS 796 compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); 797 #else 798 (void)compressionLevel; 799 EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", 800 srcFileName); 801 #endif 802 goto finish; 803 804 case FIO_lz4Compression: 805 #ifdef ZSTD_LZ4COMPRESS 806 compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 807 #else 808 (void)compressionLevel; 809 EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", 810 srcFileName); 811 #endif 812 goto finish; 813 } 814 815 /* init */ 816 #ifdef ZSTD_NEWAPI 817 if (fileSize!=0) /* when src is stdin, fileSize==0, but is effectively unknown */ 818 ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); /* note : fileSize==0 means "empty" */ 819 #elif defined(ZSTD_MULTITHREAD) 820 CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */ 821 #else 822 CHECK( ZSTD_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */ 823 #endif 824 825 /* Main compression loop */ 826 while (1) { 827 /* Fill input Buffer */ 828 size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); 829 ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; 830 if (inSize==0) break; 831 readsize += inSize; 832 833 while (inBuff.pos != inBuff.size) { 834 ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; 835 #ifdef ZSTD_NEWAPI 836 CHECK( ZSTD_compress_generic(ress.cctx, 837 &outBuff, &inBuff, ZSTD_e_continue) ); 838 #elif defined(ZSTD_MULTITHREAD) 839 CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) ); 840 #else 841 CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) ); 842 #endif 843 844 /* Write compressed stream */ 845 if (outBuff.pos) { 846 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); 847 if (sizeCheck!=outBuff.pos) 848 EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); 849 compressedfilesize += outBuff.pos; 850 } } 851 if (g_nbThreads > 1) { 852 if (!fileSize) 853 DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20)) 854 else 855 DISPLAYUPDATE(2, "\rRead : %u / %u MB", 856 (U32)(readsize>>20), (U32)(fileSize>>20)); 857 } else { 858 if (!fileSize) 859 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 860 (U32)(readsize>>20), 861 (double)compressedfilesize/readsize*100) 862 else 863 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 864 (U32)(readsize>>20), (U32)(fileSize>>20), 865 (double)compressedfilesize/readsize*100); 866 } 867 } 868 869 /* End of Frame */ 870 { size_t result = 1; 871 while (result != 0) { 872 ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; 873 #ifdef ZSTD_NEWAPI 874 ZSTD_inBuffer inBuff = { NULL, 0, 0 }; 875 result = ZSTD_compress_generic(ress.cctx, 876 &outBuff, &inBuff, ZSTD_e_end); 877 #elif defined(ZSTD_MULTITHREAD) 878 result = ZSTDMT_endStream(ress.cctx, &outBuff); 879 #else 880 result = ZSTD_endStream(ress.cctx, &outBuff); 881 #endif 882 if (ZSTD_isError(result)) { 883 EXM_THROW(26, "Compression error during frame end : %s", 884 ZSTD_getErrorName(result)); 885 } 886 { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); 887 if (sizeCheck!=outBuff.pos) 888 EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); 889 } 890 compressedfilesize += outBuff.pos; 891 } 892 } 893 894 finish: 895 /* Status */ 896 DISPLAYLEVEL(2, "\r%79s\r", ""); 897 DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, 898 (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100, 899 (unsigned long long)readsize, (unsigned long long) compressedfilesize, 900 dstFileName); 901 902 return 0; 903 } 904 905 906 /*! FIO_compressFilename_srcFile() : 907 * note : ress.destFile already opened 908 * @return : 0 : compression completed correctly, 909 * 1 : missing or pb opening srcFileName 910 */ 911 static int FIO_compressFilename_srcFile(cRess_t ress, 912 const char* dstFileName, const char* srcFileName, 913 int compressionLevel) 914 { 915 int result; 916 917 /* File check */ 918 if (UTIL_isDirectory(srcFileName)) { 919 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 920 return 1; 921 } 922 923 ress.srcFile = FIO_openSrcFile(srcFileName); 924 if (!ress.srcFile) return 1; /* srcFile could not be opened */ 925 926 result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); 927 928 fclose(ress.srcFile); 929 if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) { 930 if (remove(srcFileName)) 931 EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); 932 } 933 return result; 934 } 935 936 937 /*! FIO_compressFilename_dstFile() : 938 * @return : 0 : compression completed correctly, 939 * 1 : pb 940 */ 941 static int FIO_compressFilename_dstFile(cRess_t ress, 942 const char* dstFileName, 943 const char* srcFileName, 944 int compressionLevel) 945 { 946 int result; 947 stat_t statbuf; 948 int stat_result = 0; 949 950 ress.dstFile = FIO_openDstFile(dstFileName); 951 if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ 952 953 if (UTIL_isRegularFile(dstFileName)) { 954 g_artefact = dstFileName; 955 signal(SIGINT, INThandler); 956 } else { 957 g_artefact = NULL; 958 } 959 960 961 if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) 962 stat_result = 1; 963 result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); 964 965 if (fclose(ress.dstFile)) { /* error closing dstFile */ 966 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 967 result=1; 968 } 969 if (result!=0) { /* remove operation artefact */ 970 if (remove(dstFileName)) 971 EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); 972 } 973 else if (strcmp (dstFileName, stdoutmark) && stat_result) 974 UTIL_setFileStat(dstFileName, &statbuf); 975 976 signal(SIGINT, SIG_DFL); 977 978 return result; 979 } 980 981 982 int FIO_compressFilename(const char* dstFileName, const char* srcFileName, 983 const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams) 984 { 985 clock_t const start = clock(); 986 U64 const srcSize = UTIL_getFileSize(srcFileName); 987 988 cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); 989 int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); 990 991 double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; 992 DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); 993 994 FIO_freeCResources(ress); 995 return result; 996 } 997 998 999 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, 1000 const char* suffix, 1001 const char* dictFileName, int compressionLevel, 1002 ZSTD_compressionParameters* comprParams) 1003 { 1004 int missed_files = 0; 1005 size_t dfnSize = FNSPACE; 1006 char* dstFileName = (char*)malloc(FNSPACE); 1007 size_t const suffixSize = suffix ? strlen(suffix) : 0; 1008 U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; 1009 cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); 1010 1011 /* init */ 1012 if (dstFileName==NULL) 1013 EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); 1014 if (suffix == NULL) 1015 EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown"); /* should never happen */ 1016 1017 /* loop on each file */ 1018 if (!strcmp(suffix, stdoutmark)) { 1019 unsigned u; 1020 ress.dstFile = stdout; 1021 SET_BINARY_MODE(stdout); 1022 for (u=0; u<nbFiles; u++) 1023 missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u], compressionLevel); 1024 if (fclose(ress.dstFile)) 1025 EXM_THROW(29, "Write error : cannot properly close stdout"); 1026 } else { 1027 unsigned u; 1028 for (u=0; u<nbFiles; u++) { 1029 size_t const ifnSize = strlen(inFileNamesTable[u]); 1030 if (dfnSize <= ifnSize+suffixSize+1) { /* resize name buffer */ 1031 free(dstFileName); 1032 dfnSize = ifnSize + 20; 1033 dstFileName = (char*)malloc(dfnSize); 1034 if (!dstFileName) 1035 EXM_THROW(30, "zstd: %s", strerror(errno)); 1036 } 1037 strcpy(dstFileName, inFileNamesTable[u]); 1038 strcat(dstFileName, suffix); 1039 missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel); 1040 } } 1041 1042 FIO_freeCResources(ress); 1043 free(dstFileName); 1044 return missed_files; 1045 } 1046 1047 #endif /* #ifndef ZSTD_NOCOMPRESS */ 1048 1049 1050 1051 #ifndef ZSTD_NODECOMPRESS 1052 1053 /* ************************************************************************** 1054 * Decompression 1055 ***************************************************************************/ 1056 typedef struct { 1057 void* srcBuffer; 1058 size_t srcBufferSize; 1059 size_t srcBufferLoaded; 1060 void* dstBuffer; 1061 size_t dstBufferSize; 1062 ZSTD_DStream* dctx; 1063 FILE* dstFile; 1064 } dRess_t; 1065 1066 static dRess_t FIO_createDResources(const char* dictFileName) 1067 { 1068 dRess_t ress; 1069 memset(&ress, 0, sizeof(ress)); 1070 1071 /* Allocation */ 1072 ress.dctx = ZSTD_createDStream(); 1073 if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); 1074 CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) ); 1075 ress.srcBufferSize = ZSTD_DStreamInSize(); 1076 ress.srcBuffer = malloc(ress.srcBufferSize); 1077 ress.dstBufferSize = ZSTD_DStreamOutSize(); 1078 ress.dstBuffer = malloc(ress.dstBufferSize); 1079 if (!ress.srcBuffer || !ress.dstBuffer) 1080 EXM_THROW(61, "Allocation error : not enough memory"); 1081 1082 /* dictionary */ 1083 { void* dictBuffer; 1084 size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName); 1085 CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); 1086 free(dictBuffer); 1087 } 1088 1089 return ress; 1090 } 1091 1092 static void FIO_freeDResources(dRess_t ress) 1093 { 1094 CHECK( ZSTD_freeDStream(ress.dctx) ); 1095 free(ress.srcBuffer); 1096 free(ress.dstBuffer); 1097 } 1098 1099 1100 /** FIO_fwriteSparse() : 1101 * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ 1102 static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) 1103 { 1104 const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 1105 size_t bufferSizeT = bufferSize / sizeof(size_t); 1106 const size_t* const bufferTEnd = bufferT + bufferSizeT; 1107 const size_t* ptrT = bufferT; 1108 static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ 1109 1110 if (!g_sparseFileSupport) { /* normal write */ 1111 size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); 1112 if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); 1113 return 0; 1114 } 1115 1116 /* avoid int overflow */ 1117 if (storedSkips > 1 GB) { 1118 int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); 1119 if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); 1120 storedSkips -= 1 GB; 1121 } 1122 1123 while (ptrT < bufferTEnd) { 1124 size_t seg0SizeT = segmentSizeT; 1125 size_t nb0T; 1126 1127 /* count leading zeros */ 1128 if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 1129 bufferSizeT -= seg0SizeT; 1130 for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 1131 storedSkips += (unsigned)(nb0T * sizeof(size_t)); 1132 1133 if (nb0T != seg0SizeT) { /* not all 0s */ 1134 int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1135 if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); 1136 storedSkips = 0; 1137 seg0SizeT -= nb0T; 1138 ptrT += nb0T; 1139 { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); 1140 if (sizeCheck != seg0SizeT) 1141 EXM_THROW(73, "Write error : cannot write decoded block"); 1142 } } 1143 ptrT += seg0SizeT; 1144 } 1145 1146 { static size_t const maskT = sizeof(size_t)-1; 1147 if (bufferSize & maskT) { 1148 /* size not multiple of sizeof(size_t) : implies end of block */ 1149 const char* const restStart = (const char*)bufferTEnd; 1150 const char* restPtr = restStart; 1151 size_t restSize = bufferSize & maskT; 1152 const char* const restEnd = restStart + restSize; 1153 for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 1154 storedSkips += (unsigned) (restPtr - restStart); 1155 if (restPtr != restEnd) { 1156 int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1157 if (seekResult) 1158 EXM_THROW(74, "Sparse skip error ; try --no-sparse"); 1159 storedSkips = 0; 1160 { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); 1161 if (sizeCheck != (size_t)(restEnd - restPtr)) 1162 EXM_THROW(75, "Write error : cannot write decoded end of block"); 1163 } } } } 1164 1165 return storedSkips; 1166 } 1167 1168 static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) 1169 { 1170 if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ 1171 int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1172 if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)"); 1173 { const char lastZeroByte[1] = { 0 }; 1174 size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); 1175 if (sizeCheck != 1) 1176 EXM_THROW(69, "Write error : cannot write last zero"); 1177 } } 1178 } 1179 1180 1181 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode 1182 @return : 0 (no error) */ 1183 static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) 1184 { 1185 size_t const blockSize = MIN(64 KB, bufferSize); 1186 size_t readFromInput = 1; 1187 unsigned storedSkips = 0; 1188 1189 /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ 1190 { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); 1191 if (sizeCheck != alreadyLoaded) { 1192 DISPLAYLEVEL(1, "Pass-through write error \n"); 1193 return 1; 1194 } } 1195 1196 while (readFromInput) { 1197 readFromInput = fread(buffer, 1, blockSize, finput); 1198 storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips); 1199 } 1200 1201 FIO_fwriteSparseEnd(foutput, storedSkips); 1202 return 0; 1203 } 1204 1205 static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName) 1206 { 1207 ZSTD_frameHeader header; 1208 /* No special help for these errors */ 1209 if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) 1210 return; 1211 /* Try to decode the frame header */ 1212 ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); 1213 if (ret == 0) { 1214 U32 const windowSize = (U32)header.windowSize; 1215 U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0); 1216 U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1)); 1217 assert(header.windowSize <= (U64)((U32)-1)); 1218 assert(g_memLimit > 0); 1219 DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n", 1220 srcFileName, header.windowSize, g_memLimit); 1221 if (windowLog <= ZSTD_WINDOWLOG_MAX) { 1222 DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n", 1223 srcFileName, windowLog, windowMB); 1224 return; 1225 } 1226 } else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) { 1227 DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n", 1228 srcFileName, ZSTD_getErrorName(ret)); 1229 return; 1230 } 1231 DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n", 1232 srcFileName, ZSTD_WINDOWLOG_MAX); 1233 } 1234 1235 /** FIO_decompressFrame() : 1236 * @return : size of decoded zstd frame, or an error code 1237 */ 1238 #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) 1239 unsigned long long FIO_decompressZstdFrame(dRess_t* ress, 1240 FILE* finput, 1241 const char* srcFileName, 1242 U64 alreadyDecoded) 1243 { 1244 U64 frameSize = 0; 1245 U32 storedSkips = 0; 1246 1247 size_t const srcFileLength = strlen(srcFileName); 1248 if (srcFileLength>20) srcFileName += srcFileLength-20; /* display last 20 characters only */ 1249 1250 ZSTD_resetDStream(ress->dctx); 1251 1252 /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ 1253 { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; 1254 if (ress->srcBufferLoaded < toDecode) { 1255 size_t const toRead = toDecode - ress->srcBufferLoaded; 1256 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1257 ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); 1258 } } 1259 1260 /* Main decompression Loop */ 1261 while (1) { 1262 ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; 1263 ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; 1264 size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); 1265 if (ZSTD_isError(readSizeHint)) { 1266 DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", 1267 srcFileName, ZSTD_getErrorName(readSizeHint)); 1268 FIO_zstdErrorHelp(ress, readSizeHint, srcFileName); 1269 return FIO_ERROR_FRAME_DECODING; 1270 } 1271 1272 /* Write block */ 1273 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); 1274 frameSize += outBuff.pos; 1275 DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", 1276 srcFileName, (U32)((alreadyDecoded+frameSize)>>20) ); 1277 1278 if (inBuff.pos > 0) { 1279 memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); 1280 ress->srcBufferLoaded -= inBuff.pos; 1281 } 1282 1283 if (readSizeHint == 0) break; /* end of frame */ 1284 if (inBuff.size != inBuff.pos) { 1285 DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n", 1286 srcFileName); 1287 return FIO_ERROR_FRAME_DECODING; 1288 } 1289 1290 /* Fill input buffer */ 1291 { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ 1292 if (ress->srcBufferLoaded < toDecode) { 1293 size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ 1294 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1295 size_t const readSize = fread(startPosition, 1, toRead, finput); 1296 if (readSize==0) { 1297 DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", 1298 srcFileName); 1299 return FIO_ERROR_FRAME_DECODING; 1300 } 1301 ress->srcBufferLoaded += readSize; 1302 } } } 1303 1304 FIO_fwriteSparseEnd(ress->dstFile, storedSkips); 1305 1306 return frameSize; 1307 } 1308 1309 1310 #ifdef ZSTD_GZDECOMPRESS 1311 static unsigned long long FIO_decompressGzFrame(dRess_t* ress, 1312 FILE* srcFile, const char* srcFileName) 1313 { 1314 unsigned long long outFileSize = 0; 1315 z_stream strm; 1316 int flush = Z_NO_FLUSH; 1317 int decodingError = 0; 1318 1319 strm.zalloc = Z_NULL; 1320 strm.zfree = Z_NULL; 1321 strm.opaque = Z_NULL; 1322 strm.next_in = 0; 1323 strm.avail_in = 0; 1324 /* see http://www.zlib.net/manual.html */ 1325 if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) 1326 return FIO_ERROR_FRAME_DECODING; 1327 1328 strm.next_out = (Bytef*)ress->dstBuffer; 1329 strm.avail_out = (uInt)ress->dstBufferSize; 1330 strm.avail_in = (uInt)ress->srcBufferLoaded; 1331 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1332 1333 for ( ; ; ) { 1334 int ret; 1335 if (strm.avail_in == 0) { 1336 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 1337 if (ress->srcBufferLoaded == 0) flush = Z_FINISH; 1338 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1339 strm.avail_in = (uInt)ress->srcBufferLoaded; 1340 } 1341 ret = inflate(&strm, flush); 1342 if (ret == Z_BUF_ERROR) { 1343 DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); 1344 decodingError = 1; break; 1345 } 1346 if (ret != Z_OK && ret != Z_STREAM_END) { 1347 DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); 1348 decodingError = 1; break; 1349 } 1350 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 1351 if (decompBytes) { 1352 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { 1353 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1354 decodingError = 1; break; 1355 } 1356 outFileSize += decompBytes; 1357 strm.next_out = (Bytef*)ress->dstBuffer; 1358 strm.avail_out = (uInt)ress->dstBufferSize; 1359 } 1360 } 1361 if (ret == Z_STREAM_END) break; 1362 } 1363 1364 if (strm.avail_in > 0) 1365 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 1366 ress->srcBufferLoaded = strm.avail_in; 1367 if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ 1368 && (decodingError==0) ) { 1369 DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); 1370 decodingError = 1; 1371 } 1372 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 1373 } 1374 #endif 1375 1376 1377 #ifdef ZSTD_LZMADECOMPRESS 1378 static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) 1379 { 1380 unsigned long long outFileSize = 0; 1381 lzma_stream strm = LZMA_STREAM_INIT; 1382 lzma_action action = LZMA_RUN; 1383 lzma_ret initRet; 1384 int decodingError = 0; 1385 1386 strm.next_in = 0; 1387 strm.avail_in = 0; 1388 if (plain_lzma) { 1389 initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ 1390 } else { 1391 initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ 1392 } 1393 1394 if (initRet != LZMA_OK) { 1395 DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", 1396 plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", 1397 srcFileName, initRet); 1398 return FIO_ERROR_FRAME_DECODING; 1399 } 1400 1401 strm.next_out = (BYTE*)ress->dstBuffer; 1402 strm.avail_out = ress->dstBufferSize; 1403 strm.next_in = (BYTE const*)ress->srcBuffer; 1404 strm.avail_in = ress->srcBufferLoaded; 1405 1406 for ( ; ; ) { 1407 lzma_ret ret; 1408 if (strm.avail_in == 0) { 1409 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 1410 if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; 1411 strm.next_in = (BYTE const*)ress->srcBuffer; 1412 strm.avail_in = ress->srcBufferLoaded; 1413 } 1414 ret = lzma_code(&strm, action); 1415 1416 if (ret == LZMA_BUF_ERROR) { 1417 DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); 1418 decodingError = 1; break; 1419 } 1420 if (ret != LZMA_OK && ret != LZMA_STREAM_END) { 1421 DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", 1422 srcFileName, ret); 1423 decodingError = 1; break; 1424 } 1425 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 1426 if (decompBytes) { 1427 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { 1428 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1429 decodingError = 1; break; 1430 } 1431 outFileSize += decompBytes; 1432 strm.next_out = (BYTE*)ress->dstBuffer; 1433 strm.avail_out = ress->dstBufferSize; 1434 } } 1435 if (ret == LZMA_STREAM_END) break; 1436 } 1437 1438 if (strm.avail_in > 0) 1439 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 1440 ress->srcBufferLoaded = strm.avail_in; 1441 lzma_end(&strm); 1442 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 1443 } 1444 #endif 1445 1446 #ifdef ZSTD_LZ4DECOMPRESS 1447 static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, 1448 FILE* srcFile, const char* srcFileName) 1449 { 1450 unsigned long long filesize = 0; 1451 LZ4F_errorCode_t nextToLoad; 1452 LZ4F_decompressionContext_t dCtx; 1453 LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 1454 int decodingError = 0; 1455 1456 if (LZ4F_isError(errorCode)) { 1457 DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); 1458 return FIO_ERROR_FRAME_DECODING; 1459 } 1460 1461 /* Init feed with magic number (already consumed from FILE* sFile) */ 1462 { size_t inSize = 4; 1463 size_t outSize= 0; 1464 MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); 1465 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); 1466 if (LZ4F_isError(nextToLoad)) { 1467 DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", 1468 srcFileName, LZ4F_getErrorName(nextToLoad)); 1469 LZ4F_freeDecompressionContext(dCtx); 1470 return FIO_ERROR_FRAME_DECODING; 1471 } } 1472 1473 /* Main Loop */ 1474 for (;nextToLoad;) { 1475 size_t readSize; 1476 size_t pos = 0; 1477 size_t decodedBytes = ress->dstBufferSize; 1478 1479 /* Read input */ 1480 if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; 1481 readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); 1482 if (!readSize) break; /* reached end of file or stream */ 1483 1484 while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ 1485 /* Decode Input (at least partially) */ 1486 size_t remaining = readSize - pos; 1487 decodedBytes = ress->dstBufferSize; 1488 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); 1489 if (LZ4F_isError(nextToLoad)) { 1490 DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", 1491 srcFileName, LZ4F_getErrorName(nextToLoad)); 1492 decodingError = 1; break; 1493 } 1494 pos += remaining; 1495 1496 /* Write Block */ 1497 if (decodedBytes) { 1498 if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { 1499 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1500 decodingError = 1; break; 1501 } 1502 filesize += decodedBytes; 1503 DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); 1504 } 1505 1506 if (!nextToLoad) break; 1507 } 1508 } 1509 /* can be out because readSize == 0, which could be an fread() error */ 1510 if (ferror(srcFile)) { 1511 DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); 1512 decodingError=1; 1513 } 1514 1515 if (nextToLoad!=0) { 1516 DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); 1517 decodingError=1; 1518 } 1519 1520 LZ4F_freeDecompressionContext(dCtx); 1521 ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ 1522 1523 return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; 1524 } 1525 #endif 1526 1527 1528 1529 /** FIO_decompressFrames() : 1530 * Find and decode frames inside srcFile 1531 * srcFile presumed opened and valid 1532 * @return : 0 : OK 1533 * 1 : error 1534 */ 1535 static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, 1536 const char* dstFileName, const char* srcFileName) 1537 { 1538 unsigned readSomething = 0; 1539 unsigned long long filesize = 0; 1540 assert(srcFile != NULL); 1541 1542 /* for each frame */ 1543 for ( ; ; ) { 1544 /* check magic number -> version */ 1545 size_t const toRead = 4; 1546 const BYTE* const buf = (const BYTE*)ress.srcBuffer; 1547 if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ 1548 ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, 1549 (size_t)1, toRead - ress.srcBufferLoaded, srcFile); 1550 if (ress.srcBufferLoaded==0) { 1551 if (readSomething==0) { /* srcFile is empty (which is invalid) */ 1552 DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); 1553 return 1; 1554 } /* else, just reached frame boundary */ 1555 break; /* no more input */ 1556 } 1557 readSomething = 1; /* there is at least 1 byte in srcFile */ 1558 if (ress.srcBufferLoaded < toRead) { 1559 DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); 1560 return 1; 1561 } 1562 if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { 1563 unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize); 1564 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1565 filesize += frameSize; 1566 } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ 1567 #ifdef ZSTD_GZDECOMPRESS 1568 unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); 1569 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1570 filesize += frameSize; 1571 #else 1572 DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); 1573 return 1; 1574 #endif 1575 } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ 1576 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ 1577 #ifdef ZSTD_LZMADECOMPRESS 1578 unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); 1579 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1580 filesize += frameSize; 1581 #else 1582 DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); 1583 return 1; 1584 #endif 1585 } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { 1586 #ifdef ZSTD_LZ4DECOMPRESS 1587 unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); 1588 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1589 filesize += frameSize; 1590 #else 1591 DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); 1592 return 1; 1593 #endif 1594 } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ 1595 return FIO_passThrough(ress.dstFile, srcFile, 1596 ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); 1597 } else { 1598 DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); 1599 return 1; 1600 } } /* for each frame */ 1601 1602 /* Final Status */ 1603 DISPLAYLEVEL(2, "\r%79s\r", ""); 1604 DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); 1605 1606 return 0; 1607 } 1608 1609 1610 /** FIO_decompressSrcFile() : 1611 Decompression `srcFileName` into `ress.dstFile` 1612 @return : 0 : OK 1613 1 : operation not started 1614 */ 1615 static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) 1616 { 1617 FILE* srcFile; 1618 int result; 1619 1620 if (UTIL_isDirectory(srcFileName)) { 1621 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 1622 return 1; 1623 } 1624 1625 srcFile = FIO_openSrcFile(srcFileName); 1626 if (srcFile==NULL) return 1; 1627 1628 result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName); 1629 1630 /* Close file */ 1631 if (fclose(srcFile)) { 1632 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ 1633 return 1; 1634 } 1635 if ( g_removeSrcFile /* --rm */ 1636 && (result==0) /* decompression successful */ 1637 && strcmp(srcFileName, stdinmark) ) /* not stdin */ { 1638 if (remove(srcFileName)) { 1639 /* failed to remove src file */ 1640 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 1641 return 1; 1642 } } 1643 return result; 1644 } 1645 1646 1647 /** FIO_decompressFile_extRess() : 1648 decompress `srcFileName` into `dstFileName` 1649 @return : 0 : OK 1650 1 : operation aborted (src not available, dst already taken, etc.) 1651 */ 1652 static int FIO_decompressDstFile(dRess_t ress, 1653 const char* dstFileName, const char* srcFileName) 1654 { 1655 int result; 1656 stat_t statbuf; 1657 int stat_result = 0; 1658 1659 ress.dstFile = FIO_openDstFile(dstFileName); 1660 if (ress.dstFile==0) return 1; 1661 1662 if (UTIL_isRegularFile(dstFileName)) { 1663 g_artefact = dstFileName; 1664 signal(SIGINT, INThandler); 1665 } else { 1666 g_artefact = NULL; 1667 } 1668 1669 if ( strcmp(srcFileName, stdinmark) 1670 && UTIL_getFileStat(srcFileName, &statbuf) ) 1671 stat_result = 1; 1672 result = FIO_decompressSrcFile(ress, dstFileName, srcFileName); 1673 1674 if (fclose(ress.dstFile)) { 1675 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 1676 result = 1; 1677 } 1678 1679 if ( (result != 0) /* operation failure */ 1680 && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ 1681 && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */ 1682 remove(dstFileName); /* remove decompression artefact; note don't do anything special if remove() fails */ 1683 else { /* operation success */ 1684 if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */ 1685 && strcmp(dstFileName, nulmark) /* special case : don't chmod /dev/null */ 1686 && stat_result ) /* file permissions correctly extracted from src */ 1687 UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ 1688 } 1689 1690 signal(SIGINT, SIG_DFL); 1691 1692 return result; 1693 } 1694 1695 1696 int FIO_decompressFilename(const char* dstFileName, const char* srcFileName, 1697 const char* dictFileName) 1698 { 1699 dRess_t const ress = FIO_createDResources(dictFileName); 1700 1701 int const decodingError = FIO_decompressDstFile(ress, dstFileName, srcFileName); 1702 1703 FIO_freeDResources(ress); 1704 return decodingError; 1705 } 1706 1707 1708 #define MAXSUFFIXSIZE 8 1709 int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, 1710 const char* suffix, 1711 const char* dictFileName) 1712 { 1713 int skippedFiles = 0; 1714 int missingFiles = 0; 1715 dRess_t ress = FIO_createDResources(dictFileName); 1716 1717 if (suffix==NULL) 1718 EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */ 1719 1720 if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */ 1721 unsigned u; 1722 ress.dstFile = FIO_openDstFile(suffix); 1723 if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix); 1724 for (u=0; u<nbFiles; u++) 1725 missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]); 1726 if (fclose(ress.dstFile)) 1727 EXM_THROW(72, "Write error : cannot properly close stdout"); 1728 } else { 1729 size_t suffixSize; 1730 size_t dfnSize = FNSPACE; 1731 unsigned u; 1732 char* dstFileName = (char*)malloc(FNSPACE); 1733 if (dstFileName==NULL) 1734 EXM_THROW(73, "not enough memory for dstFileName"); 1735 for (u=0; u<nbFiles; u++) { /* create dstFileName */ 1736 const char* const srcFileName = srcNamesTable[u]; 1737 const char* const suffixPtr = strrchr(srcFileName, '.'); 1738 size_t const sfnSize = strlen(srcFileName); 1739 if (!suffixPtr) { 1740 DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", 1741 srcFileName); 1742 skippedFiles++; 1743 continue; 1744 } 1745 suffixSize = strlen(suffixPtr); 1746 if (dfnSize+suffixSize <= sfnSize+1) { 1747 free(dstFileName); 1748 dfnSize = sfnSize + 20; 1749 dstFileName = (char*)malloc(dfnSize); 1750 if (dstFileName==NULL) 1751 EXM_THROW(74, "not enough memory for dstFileName"); 1752 } 1753 if (sfnSize <= suffixSize 1754 || (strcmp(suffixPtr, GZ_EXTENSION) 1755 && strcmp(suffixPtr, XZ_EXTENSION) 1756 && strcmp(suffixPtr, ZSTD_EXTENSION) 1757 && strcmp(suffixPtr, LZMA_EXTENSION) 1758 && strcmp(suffixPtr, LZ4_EXTENSION)) ) { 1759 DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s/%s expected) -- ignored \n", 1760 srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION, LZ4_EXTENSION); 1761 skippedFiles++; 1762 continue; 1763 } else { 1764 memcpy(dstFileName, srcFileName, sfnSize - suffixSize); 1765 dstFileName[sfnSize-suffixSize] = '\0'; 1766 } 1767 missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName); 1768 } 1769 free(dstFileName); 1770 } 1771 1772 FIO_freeDResources(ress); 1773 return missingFiles + skippedFiles; 1774 } 1775 1776 1777 1778 /* ************************************************************************** 1779 * .zst file info (--list command) 1780 ***************************************************************************/ 1781 1782 typedef struct { 1783 U64 decompressedSize; 1784 U64 compressedSize; 1785 U64 windowSize; 1786 int numActualFrames; 1787 int numSkippableFrames; 1788 int decompUnavailable; 1789 int usesCheck; 1790 U32 nbFiles; 1791 } fileInfo_t; 1792 1793 /** getFileInfo() : 1794 * Reads information from file, stores in *info 1795 * @return : 0 if successful 1796 * 1 for frame analysis error 1797 * 2 for file not compressed with zstd 1798 * 3 for cases in which file could not be opened. 1799 */ 1800 static int getFileInfo(fileInfo_t* info, const char* inFileName){ 1801 int detectError = 0; 1802 FILE* const srcFile = FIO_openSrcFile(inFileName); 1803 if (srcFile == NULL) { 1804 DISPLAY("Error: could not open source file %s\n", inFileName); 1805 return 3; 1806 } 1807 info->compressedSize = UTIL_getFileSize(inFileName); 1808 1809 /* begin analyzing frame */ 1810 for ( ; ; ) { 1811 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; 1812 size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); 1813 if (numBytesRead < ZSTD_frameHeaderSize_min) { 1814 if ( feof(srcFile) 1815 && (numBytesRead == 0) 1816 && (info->compressedSize > 0) ) { 1817 break; 1818 } 1819 else if (feof(srcFile)) { 1820 DISPLAY("Error: reached end of file with incomplete frame\n"); 1821 detectError = 2; 1822 break; 1823 } 1824 else { 1825 DISPLAY("Error: did not reach end of file but ran out of frames\n"); 1826 detectError = 1; 1827 break; 1828 } 1829 } 1830 { U32 const magicNumber = MEM_readLE32(headerBuffer); 1831 /* Zstandard frame */ 1832 if (magicNumber == ZSTD_MAGICNUMBER) { 1833 ZSTD_frameHeader header; 1834 U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); 1835 if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { 1836 info->decompUnavailable = 1; 1837 } else { 1838 info->decompressedSize += frameContentSize; 1839 } 1840 if (ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0) { 1841 DISPLAY("Error: could not decode frame header\n"); 1842 detectError = 1; 1843 break; 1844 } 1845 info->windowSize = header.windowSize; 1846 /* move to the end of the frame header */ 1847 { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); 1848 if (ZSTD_isError(headerSize)) { 1849 DISPLAY("Error: could not determine frame header size\n"); 1850 detectError = 1; 1851 break; 1852 } 1853 { int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); 1854 if (ret != 0) { 1855 DISPLAY("Error: could not move to end of frame header\n"); 1856 detectError = 1; 1857 break; 1858 } } } 1859 1860 /* skip the rest of the blocks in the frame */ 1861 { int lastBlock = 0; 1862 do { 1863 BYTE blockHeaderBuffer[3]; 1864 size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); 1865 if (readBytes != 3) { 1866 DISPLAY("There was a problem reading the block header\n"); 1867 detectError = 1; 1868 break; 1869 } 1870 { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); 1871 U32 const blockTypeID = (blockHeader >> 1) & 3; 1872 U32 const isRLE = (blockTypeID == 1); 1873 U32 const isWrongBlock = (blockTypeID == 3); 1874 long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); 1875 if (isWrongBlock) { 1876 DISPLAY("Error: unsupported block type \n"); 1877 detectError = 1; 1878 break; 1879 } 1880 lastBlock = blockHeader & 1; 1881 { int const ret = fseek(srcFile, blockSize, SEEK_CUR); 1882 if (ret != 0) { 1883 DISPLAY("Error: could not skip to end of block\n"); 1884 detectError = 1; 1885 break; 1886 } } } 1887 } while (lastBlock != 1); 1888 1889 if (detectError) break; 1890 } 1891 1892 /* check if checksum is used */ 1893 { BYTE const frameHeaderDescriptor = headerBuffer[4]; 1894 int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; 1895 if (contentChecksumFlag) { 1896 int const ret = fseek(srcFile, 4, SEEK_CUR); 1897 info->usesCheck = 1; 1898 if (ret != 0) { 1899 DISPLAY("Error: could not skip past checksum\n"); 1900 detectError = 1; 1901 break; 1902 } } } 1903 info->numActualFrames++; 1904 } 1905 /* Skippable frame */ 1906 else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { 1907 U32 const frameSize = MEM_readLE32(headerBuffer + 4); 1908 long const seek = (long)(8 + frameSize - numBytesRead); 1909 int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR); 1910 if (ret != 0) { 1911 DISPLAY("Error: could not find end of skippable frame\n"); 1912 detectError = 1; 1913 break; 1914 } 1915 info->numSkippableFrames++; 1916 } 1917 /* unknown content */ 1918 else { 1919 detectError = 2; 1920 break; 1921 } 1922 } 1923 } /* end analyzing frame */ 1924 fclose(srcFile); 1925 info->nbFiles = 1; 1926 return detectError; 1927 } 1928 1929 static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ 1930 unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); 1931 const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; 1932 double const windowSizeUnit = (double)info->windowSize / unit; 1933 double const compressedSizeUnit = (double)info->compressedSize / unit; 1934 double const decompressedSizeUnit = (double)info->decompressedSize / unit; 1935 double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; 1936 const char* const checkString = (info->usesCheck ? "XXH64" : "None"); 1937 if (displayLevel <= 2) { 1938 if (!info->decompUnavailable) { 1939 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", 1940 info->numSkippableFrames + info->numActualFrames, 1941 info->numSkippableFrames, 1942 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 1943 ratio, checkString, inFileName); 1944 } else { 1945 DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n", 1946 info->numSkippableFrames + info->numActualFrames, 1947 info->numSkippableFrames, 1948 compressedSizeUnit, unitStr, 1949 checkString, inFileName); 1950 } 1951 } else { 1952 DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); 1953 DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); 1954 DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", 1955 windowSizeUnit, unitStr, 1956 (unsigned long long)info->windowSize); 1957 DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", 1958 compressedSizeUnit, unitStr, 1959 (unsigned long long)info->compressedSize); 1960 if (!info->decompUnavailable) { 1961 DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", 1962 decompressedSizeUnit, unitStr, 1963 (unsigned long long)info->decompressedSize); 1964 DISPLAYOUT("Ratio: %.4f\n", ratio); 1965 } 1966 DISPLAYOUT("Check: %s\n", checkString); 1967 DISPLAYOUT("\n"); 1968 } 1969 } 1970 1971 static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) 1972 { 1973 fileInfo_t total; 1974 total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; 1975 total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; 1976 total.compressedSize = fi1.compressedSize + fi2.compressedSize; 1977 total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; 1978 total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; 1979 total.usesCheck = fi1.usesCheck & fi2.usesCheck; 1980 total.nbFiles = fi1.nbFiles + fi2.nbFiles; 1981 return total; 1982 } 1983 1984 static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){ 1985 /* initialize info to avoid warnings */ 1986 fileInfo_t info; 1987 memset(&info, 0, sizeof(info)); 1988 { int const error = getFileInfo(&info, inFileName); 1989 if (error == 1) { 1990 /* display error, but provide output */ 1991 DISPLAY("An error occurred while getting file info \n"); 1992 } 1993 else if (error == 2) { 1994 DISPLAYOUT("File %s not compressed by zstd \n", inFileName); 1995 if (displayLevel > 2) DISPLAYOUT("\n"); 1996 return 1; 1997 } 1998 else if (error == 3) { 1999 /* error occurred while opening the file */ 2000 if (displayLevel > 2) DISPLAYOUT("\n"); 2001 return 1; 2002 } 2003 displayInfo(inFileName, &info, displayLevel); 2004 *total = FIO_addFInfo(*total, info); 2005 return error; 2006 } 2007 } 2008 2009 int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ 2010 if (numFiles == 0) { 2011 DISPLAYOUT("No files given\n"); 2012 return 0; 2013 } 2014 if (displayLevel <= 2) { 2015 DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); 2016 } 2017 { int error = 0; 2018 unsigned u; 2019 fileInfo_t total; 2020 memset(&total, 0, sizeof(total)); 2021 total.usesCheck = 1; 2022 for (u=0; u<numFiles;u++) { 2023 error |= FIO_listFile(&total, filenameTable[u], displayLevel); 2024 } 2025 if (numFiles > 1 && displayLevel <= 2) { 2026 unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); 2027 const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; 2028 double const compressedSizeUnit = (double)total.compressedSize / unit; 2029 double const decompressedSizeUnit = (double)total.decompressedSize / unit; 2030 double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize; 2031 const char* const checkString = (total.usesCheck ? "XXH64" : ""); 2032 DISPLAYOUT("----------------------------------------------------------------- \n"); 2033 if (total.decompUnavailable) { 2034 DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n", 2035 total.numSkippableFrames + total.numActualFrames, 2036 total.numSkippableFrames, 2037 compressedSizeUnit, unitStr, 2038 checkString, total.nbFiles); 2039 } else { 2040 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n", 2041 total.numSkippableFrames + total.numActualFrames, 2042 total.numSkippableFrames, 2043 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 2044 ratio, checkString, total.nbFiles); 2045 } 2046 } 2047 return error; 2048 } 2049 } 2050 2051 2052 #endif /* #ifndef ZSTD_NODECOMPRESS */ 2053