1 /* 2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /*-************************************ 13 * Tuning parameters 14 **************************************/ 15 #ifndef ZSTDCLI_CLEVEL_DEFAULT 16 # define ZSTDCLI_CLEVEL_DEFAULT 3 17 #endif 18 19 #ifndef ZSTDCLI_CLEVEL_MAX 20 # define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */ 21 #endif 22 23 24 25 /*-************************************ 26 * Dependencies 27 **************************************/ 28 #include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */ 29 #include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ 30 #include <stdio.h> /* fprintf(), stdin, stdout, stderr */ 31 #include <string.h> /* strcmp, strlen */ 32 #include <errno.h> /* errno */ 33 #include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ 34 #ifndef ZSTD_NOBENCH 35 # include "bench.h" /* BMK_benchFiles */ 36 #endif 37 #ifndef ZSTD_NODICT 38 # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */ 39 #endif 40 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_minCLevel */ 41 #include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */ 42 43 44 /*-************************************ 45 * Constants 46 **************************************/ 47 #define COMPRESSOR_NAME "zstd command line interface" 48 #ifndef ZSTD_VERSION 49 # define ZSTD_VERSION "v" ZSTD_VERSION_STRING 50 #endif 51 #define AUTHOR "Yann Collet" 52 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR 53 54 #define ZSTD_ZSTDMT "zstdmt" 55 #define ZSTD_UNZSTD "unzstd" 56 #define ZSTD_CAT "zstdcat" 57 #define ZSTD_ZCAT "zcat" 58 #define ZSTD_GZ "gzip" 59 #define ZSTD_GUNZIP "gunzip" 60 #define ZSTD_GZCAT "gzcat" 61 #define ZSTD_LZMA "lzma" 62 #define ZSTD_UNLZMA "unlzma" 63 #define ZSTD_XZ "xz" 64 #define ZSTD_UNXZ "unxz" 65 #define ZSTD_LZ4 "lz4" 66 #define ZSTD_UNLZ4 "unlz4" 67 68 #define KB *(1 <<10) 69 #define MB *(1 <<20) 70 #define GB *(1U<<30) 71 72 #define DISPLAY_LEVEL_DEFAULT 2 73 74 static const char* g_defaultDictName = "dictionary"; 75 static const unsigned g_defaultMaxDictSize = 110 KB; 76 static const int g_defaultDictCLevel = 3; 77 static const unsigned g_defaultSelectivityLevel = 9; 78 static const unsigned g_defaultMaxWindowLog = 27; 79 #define OVERLAP_LOG_DEFAULT 9999 80 #define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */ 81 static U32 g_overlapLog = OVERLAP_LOG_DEFAULT; 82 static U32 g_ldmHashLog = 0; 83 static U32 g_ldmMinMatch = 0; 84 static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT; 85 static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT; 86 87 88 #define DEFAULT_ACCEL 1 89 90 typedef enum { cover, fastCover, legacy } dictType; 91 92 /*-************************************ 93 * Display Macros 94 **************************************/ 95 #define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__) 96 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } 97 static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ 98 static FILE* g_displayOut; 99 100 101 /*-************************************ 102 * Command Line 103 **************************************/ 104 static int usage(const char* programName) 105 { 106 DISPLAY( "Usage : \n"); 107 DISPLAY( " %s [args] [FILE(s)] [-o file] \n", programName); 108 DISPLAY( "\n"); 109 DISPLAY( "FILE : a filename \n"); 110 DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); 111 DISPLAY( "Arguments : \n"); 112 #ifndef ZSTD_NOCOMPRESS 113 DISPLAY( " -# : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT); 114 #endif 115 #ifndef ZSTD_NODECOMPRESS 116 DISPLAY( " -d : decompression \n"); 117 #endif 118 DISPLAY( " -D file: use `file` as Dictionary \n"); 119 DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); 120 DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); 121 DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); 122 DISPLAY( " -k : preserve source file(s) (default) \n"); 123 DISPLAY( " -h/-H : display help/long help and exit \n"); 124 return 0; 125 } 126 127 static int usage_advanced(const char* programName) 128 { 129 DISPLAY(WELCOME_MESSAGE); 130 usage(programName); 131 DISPLAY( "\n"); 132 DISPLAY( "Advanced arguments : \n"); 133 DISPLAY( " -V : display Version number and exit \n"); 134 DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n"); 135 DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); 136 DISPLAY( " -c : force write to standard output, even if it is the console\n"); 137 DISPLAY( " -l : print information about zstd compressed files \n"); 138 #ifndef ZSTD_NOCOMPRESS 139 DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); 140 DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); 141 DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); 142 DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n"); 143 #ifdef ZSTD_MULTITHREAD 144 DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n"); 145 DISPLAY( " -B# : select size of each job (default: 0==automatic) \n"); 146 #endif 147 DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); 148 DISPLAY( "--[no-]check : integrity check (default: enabled) \n"); 149 #endif 150 #ifdef UTIL_HAS_CREATEFILELIST 151 DISPLAY( " -r : operate recursively on directories \n"); 152 #endif 153 DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n"); 154 #ifdef ZSTD_GZCOMPRESS 155 DISPLAY( "--format=gzip : compress files to the .gz format \n"); 156 #endif 157 #ifdef ZSTD_LZMACOMPRESS 158 DISPLAY( "--format=xz : compress files to the .xz format \n"); 159 DISPLAY( "--format=lzma : compress files to the .lzma format \n"); 160 #endif 161 #ifdef ZSTD_LZ4COMPRESS 162 DISPLAY( "--format=lz4 : compress files to the .lz4 format \n"); 163 #endif 164 #ifndef ZSTD_NODECOMPRESS 165 DISPLAY( "--test : test compressed file integrity \n"); 166 #if ZSTD_SPARSE_DEFAULT 167 DISPLAY( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout)\n"); 168 #else 169 DISPLAY( "--[no-]sparse : sparse mode (default: disabled)\n"); 170 #endif 171 #endif 172 DISPLAY( " -M# : Set a memory usage limit for decompression \n"); 173 DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); 174 #ifndef ZSTD_NODICT 175 DISPLAY( "\n"); 176 DISPLAY( "Dictionary builder : \n"); 177 DISPLAY( "--train ## : create a dictionary from a training set of files \n"); 178 DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n"); 179 DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n"); 180 DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel); 181 DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); 182 DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize); 183 DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n"); 184 #endif 185 #ifndef ZSTD_NOBENCH 186 DISPLAY( "\n"); 187 DISPLAY( "Benchmark arguments : \n"); 188 DISPLAY( " -b# : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT); 189 DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); 190 DISPLAY( " -i# : minimum evaluation time in seconds (default: 3s) \n"); 191 DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); 192 DISPLAY( "--priority=rt : set process priority to real-time \n"); 193 #endif 194 return 0; 195 } 196 197 static int badusage(const char* programName) 198 { 199 DISPLAYLEVEL(1, "Incorrect parameters\n"); 200 if (g_displayLevel >= 2) usage(programName); 201 return 1; 202 } 203 204 static void waitEnter(void) 205 { 206 int unused; 207 DISPLAY("Press enter to continue...\n"); 208 unused = getchar(); 209 (void)unused; 210 } 211 212 static const char* lastNameFromPath(const char* path) 213 { 214 const char* name = path; 215 if (strrchr(name, '/')) name = strrchr(name, '/') + 1; 216 if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */ 217 return name; 218 } 219 220 /*! exeNameMatch() : 221 @return : a non-zero value if exeName matches test, excluding the extension 222 */ 223 static int exeNameMatch(const char* exeName, const char* test) 224 { 225 return !strncmp(exeName, test, strlen(test)) && 226 (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.'); 227 } 228 229 static void errorOut(const char* msg) 230 { 231 DISPLAY("%s \n", msg); exit(1); 232 } 233 234 /*! readU32FromChar() : 235 * @return : unsigned integer value read from input in `char` format. 236 * allows and interprets K, KB, KiB, M, MB and MiB suffix. 237 * Will also modify `*stringPtr`, advancing it to position where it stopped reading. 238 * Note : function will exit() program if digit sequence overflows */ 239 static unsigned readU32FromChar(const char** stringPtr) 240 { 241 const char errorMsg[] = "error: numeric value too large"; 242 unsigned result = 0; 243 while ((**stringPtr >='0') && (**stringPtr <='9')) { 244 unsigned const max = (((unsigned)(-1)) / 10) - 1; 245 if (result > max) errorOut(errorMsg); 246 result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; 247 } 248 if ((**stringPtr=='K') || (**stringPtr=='M')) { 249 unsigned const maxK = ((unsigned)(-1)) >> 10; 250 if (result > maxK) errorOut(errorMsg); 251 result <<= 10; 252 if (**stringPtr=='M') { 253 if (result > maxK) errorOut(errorMsg); 254 result <<= 10; 255 } 256 (*stringPtr)++; /* skip `K` or `M` */ 257 if (**stringPtr=='i') (*stringPtr)++; 258 if (**stringPtr=='B') (*stringPtr)++; 259 } 260 return result; 261 } 262 263 /** longCommandWArg() : 264 * check if *stringPtr is the same as longCommand. 265 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. 266 * @return 0 and doesn't modify *stringPtr otherwise. 267 */ 268 static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) 269 { 270 size_t const comSize = strlen(longCommand); 271 int const result = !strncmp(*stringPtr, longCommand, comSize); 272 if (result) *stringPtr += comSize; 273 return result; 274 } 275 276 277 #ifndef ZSTD_NODICT 278 /** 279 * parseCoverParameters() : 280 * reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params 281 * @return 1 means that cover parameters were correct 282 * @return 0 in case of malformed parameters 283 */ 284 static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params) 285 { 286 memset(params, 0, sizeof(*params)); 287 for (; ;) { 288 if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 289 if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 290 if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 291 if (longCommandWArg(&stringPtr, "split=")) { 292 unsigned splitPercentage = readU32FromChar(&stringPtr); 293 params->splitPoint = (double)splitPercentage / 100.0; 294 if (stringPtr[0]==',') { stringPtr++; continue; } else break; 295 } 296 return 0; 297 } 298 if (stringPtr[0] != 0) return 0; 299 DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100)); 300 return 1; 301 } 302 303 /** 304 * parseFastCoverParameters() : 305 * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params 306 * @return 1 means that fastcover parameters were correct 307 * @return 0 in case of malformed parameters 308 */ 309 static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params) 310 { 311 memset(params, 0, sizeof(*params)); 312 for (; ;) { 313 if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 314 if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 315 if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 316 if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 317 if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 318 if (longCommandWArg(&stringPtr, "split=")) { 319 unsigned splitPercentage = readU32FromChar(&stringPtr); 320 params->splitPoint = (double)splitPercentage / 100.0; 321 if (stringPtr[0]==',') { stringPtr++; continue; } else break; 322 } 323 return 0; 324 } 325 if (stringPtr[0] != 0) return 0; 326 DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel); 327 return 1; 328 } 329 330 /** 331 * parseLegacyParameters() : 332 * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity 333 * @return 1 means that legacy dictionary builder parameters were correct 334 * @return 0 in case of malformed parameters 335 */ 336 static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity) 337 { 338 if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; } 339 *selectivity = readU32FromChar(&stringPtr); 340 if (stringPtr[0] != 0) return 0; 341 DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity); 342 return 1; 343 } 344 345 static ZDICT_cover_params_t defaultCoverParams(void) 346 { 347 ZDICT_cover_params_t params; 348 memset(¶ms, 0, sizeof(params)); 349 params.d = 8; 350 params.steps = 4; 351 params.splitPoint = 1.0; 352 return params; 353 } 354 355 static ZDICT_fastCover_params_t defaultFastCoverParams(void) 356 { 357 ZDICT_fastCover_params_t params; 358 memset(¶ms, 0, sizeof(params)); 359 params.d = 8; 360 params.f = 20; 361 params.steps = 4; 362 params.splitPoint = 0.75; /* different from default splitPoint of cover */ 363 params.accel = DEFAULT_ACCEL; 364 return params; 365 } 366 #endif 367 368 369 /** parseAdaptParameters() : 370 * reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr. 371 * Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized. 372 * There is no guarantee that any of these values will be updated. 373 * @return 1 means that parsing was successful, 374 * @return 0 in case of malformed parameters 375 */ 376 static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr) 377 { 378 for ( ; ;) { 379 if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 380 if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 381 DISPLAYLEVEL(4, "invalid compression parameter \n"); 382 return 0; 383 } 384 if (stringPtr[0] != 0) return 0; /* check the end of string */ 385 if (*adaptMinPtr > *adaptMaxPtr) { 386 DISPLAYLEVEL(4, "incoherent adaptation limits \n"); 387 return 0; 388 } 389 return 1; 390 } 391 392 393 /** parseCompressionParameters() : 394 * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params 395 * @return 1 means that compression parameters were correct 396 * @return 0 in case of malformed parameters 397 */ 398 static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) 399 { 400 for ( ; ;) { 401 if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 402 if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 403 if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 404 if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 405 if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 406 if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 407 if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 408 if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 409 if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 410 if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 411 if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 412 if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } 413 DISPLAYLEVEL(4, "invalid compression parameter \n"); 414 return 0; 415 } 416 417 DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); 418 DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy); 419 if (stringPtr[0] != 0) return 0; /* check the end of string */ 420 return 1; 421 } 422 423 static void printVersion(void) 424 { 425 DISPLAY(WELCOME_MESSAGE); 426 /* format support */ 427 DISPLAYLEVEL(3, "*** supports: zstd"); 428 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8) 429 DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT); 430 #endif 431 #ifdef ZSTD_GZCOMPRESS 432 DISPLAYLEVEL(3, ", gzip"); 433 #endif 434 #ifdef ZSTD_LZ4COMPRESS 435 DISPLAYLEVEL(3, ", lz4"); 436 #endif 437 #ifdef ZSTD_LZMACOMPRESS 438 DISPLAYLEVEL(3, ", lzma, xz "); 439 #endif 440 DISPLAYLEVEL(3, "\n"); 441 /* posix support */ 442 #ifdef _POSIX_C_SOURCE 443 DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); 444 #endif 445 #ifdef _POSIX_VERSION 446 DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION); 447 #endif 448 #ifdef PLATFORM_POSIX_VERSION 449 DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); 450 #endif 451 } 452 453 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; 454 455 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } 456 457 #ifdef ZSTD_NOCOMPRESS 458 /* symbols from compression library are not defined and should not be invoked */ 459 # define MINCLEVEL -50 460 # define MAXCLEVEL 22 461 #else 462 # define MINCLEVEL ZSTD_minCLevel() 463 # define MAXCLEVEL ZSTD_maxCLevel() 464 #endif 465 466 int main(int argCount, const char* argv[]) 467 { 468 int argNb, 469 followLinks = 0, 470 forceStdout = 0, 471 lastCommand = 0, 472 ldmFlag = 0, 473 main_pause = 0, 474 nbWorkers = 0, 475 adapt = 0, 476 adaptMin = MINCLEVEL, 477 adaptMax = MAXCLEVEL, 478 nextArgumentIsOutFileName = 0, 479 nextArgumentIsMaxDict = 0, 480 nextArgumentIsDictID = 0, 481 nextArgumentsAreFiles = 0, 482 nextEntryIsDictionary = 0, 483 operationResult = 0, 484 separateFiles = 0, 485 setRealTimePrio = 0, 486 singleThread = 0, 487 ultra=0; 488 double compressibility = 0.5; 489 unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ 490 size_t blockSize = 0; 491 zstd_operation_mode operation = zom_compress; 492 ZSTD_compressionParameters compressionParams; 493 int cLevel = ZSTDCLI_CLEVEL_DEFAULT; 494 int cLevelLast = -1000000000; 495 unsigned recursive = 0; 496 unsigned memLimit = 0; 497 const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ 498 unsigned filenameIdx = 0; 499 const char* programName = argv[0]; 500 const char* outFileName = NULL; 501 const char* dictFileName = NULL; 502 const char* suffix = ZSTD_EXTENSION; 503 unsigned maxDictSize = g_defaultMaxDictSize; 504 unsigned dictID = 0; 505 int dictCLevel = g_defaultDictCLevel; 506 unsigned dictSelect = g_defaultSelectivityLevel; 507 #ifdef UTIL_HAS_CREATEFILELIST 508 const char** extendedFileList = NULL; 509 char* fileNamesBuf = NULL; 510 unsigned fileNamesNb; 511 #endif 512 #ifndef ZSTD_NODICT 513 ZDICT_cover_params_t coverParams = defaultCoverParams(); 514 ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams(); 515 dictType dict = fastCover; 516 #endif 517 #ifndef ZSTD_NOBENCH 518 BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); 519 #endif 520 521 522 /* init */ 523 (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ 524 (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ 525 if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } 526 filenameTable[0] = stdinmark; 527 g_displayOut = stderr; 528 programName = lastNameFromPath(programName); 529 #ifdef ZSTD_MULTITHREAD 530 nbWorkers = 1; 531 #endif 532 533 /* preset behaviors */ 534 if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; 535 if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; 536 if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ 537 if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ 538 if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ 539 if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip, also supports multiple formats */ 540 if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ 541 if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ 542 if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma, also supports multiple formats */ 543 if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ 544 if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz, also supports multiple formats */ 545 if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); } /* behave like lz4 */ 546 if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ 547 memset(&compressionParams, 0, sizeof(compressionParams)); 548 549 /* init crash handler */ 550 FIO_addAbortHandler(); 551 552 /* command switches */ 553 for (argNb=1; argNb<argCount; argNb++) { 554 const char* argument = argv[argNb]; 555 if(!argument) continue; /* Protection if argument empty */ 556 557 if (nextArgumentsAreFiles==0) { 558 /* "-" means stdin/stdout */ 559 if (!strcmp(argument, "-")){ 560 if (!filenameIdx) { 561 filenameIdx=1, filenameTable[0]=stdinmark; 562 outFileName=stdoutmark; 563 g_displayLevel-=(g_displayLevel==2); 564 continue; 565 } } 566 567 /* Decode commands (note : aggregated commands are allowed) */ 568 if (argument[0]=='-') { 569 570 if (argument[1]=='-') { 571 /* long commands (--long-word) */ 572 if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */ 573 if (!strcmp(argument, "--list")) { operation=zom_list; continue; } 574 if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } 575 if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } 576 if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } 577 if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; } 578 if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } 579 if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } 580 if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } 581 if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } 582 if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; } 583 if (!strcmp(argument, "--ultra")) { ultra=1; continue; } 584 if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } 585 if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } 586 if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } 587 if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } 588 if (!strcmp(argument, "--test")) { operation=zom_test; continue; } 589 if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } 590 if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ 591 if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ 592 if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } 593 if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } 594 if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } 595 if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } 596 if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } 597 if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } 598 if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } 599 if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; } 600 #ifdef ZSTD_GZCOMPRESS 601 if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } 602 #endif 603 #ifdef ZSTD_LZMACOMPRESS 604 if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; } 605 if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; } 606 #endif 607 #ifdef ZSTD_LZ4COMPRESS 608 if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; } 609 #endif 610 611 /* long commands with arguments */ 612 #ifndef ZSTD_NODICT 613 if (longCommandWArg(&argument, "--train-cover")) { 614 operation = zom_train; 615 if (outFileName == NULL) 616 outFileName = g_defaultDictName; 617 dict = cover; 618 /* Allow optional arguments following an = */ 619 if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); } 620 else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } 621 else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); } 622 continue; 623 } 624 if (longCommandWArg(&argument, "--train-fastcover")) { 625 operation = zom_train; 626 if (outFileName == NULL) 627 outFileName = g_defaultDictName; 628 dict = fastCover; 629 /* Allow optional arguments following an = */ 630 if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); } 631 else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } 632 else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); } 633 continue; 634 } 635 if (longCommandWArg(&argument, "--train-legacy")) { 636 operation = zom_train; 637 if (outFileName == NULL) 638 outFileName = g_defaultDictName; 639 dict = legacy; 640 /* Allow optional arguments following an = */ 641 if (*argument == 0) { continue; } 642 else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } 643 else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); } 644 continue; 645 } 646 #endif 647 if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; } 648 if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } 649 if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } 650 if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } 651 if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; } 652 if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } 653 if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; } 654 if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; } 655 if (longCommandWArg(&argument, "--long")) { 656 unsigned ldmWindowLog = 0; 657 ldmFlag = 1; 658 /* Parse optional window log */ 659 if (*argument == '=') { 660 ++argument; 661 ldmWindowLog = readU32FromChar(&argument); 662 } else if (*argument != 0) { 663 /* Invalid character following --long */ 664 CLEAN_RETURN(badusage(programName)); 665 } 666 /* Only set windowLog if not already set by --zstd */ 667 if (compressionParams.windowLog == 0) 668 compressionParams.windowLog = ldmWindowLog; 669 continue; 670 } 671 #ifndef ZSTD_NOCOMPRESS /* linking ZSTD_minCLevel() requires compression support */ 672 if (longCommandWArg(&argument, "--fast")) { 673 /* Parse optional acceleration factor */ 674 if (*argument == '=') { 675 U32 const maxFast = (U32)-ZSTD_minCLevel(); 676 U32 fastLevel; 677 ++argument; 678 fastLevel = readU32FromChar(&argument); 679 if (fastLevel > maxFast) fastLevel = maxFast; 680 if (fastLevel) { 681 dictCLevel = cLevel = -(int)fastLevel; 682 } else { 683 CLEAN_RETURN(badusage(programName)); 684 } 685 } else if (*argument != 0) { 686 /* Invalid character following --fast */ 687 CLEAN_RETURN(badusage(programName)); 688 } else { 689 cLevel = -1; /* default for --fast */ 690 } 691 continue; 692 } 693 #endif 694 /* fall-through, will trigger bad_usage() later on */ 695 } 696 697 argument++; 698 while (argument[0]!=0) { 699 if (lastCommand) { 700 DISPLAY("error : command must be followed by argument \n"); 701 CLEAN_RETURN(1); 702 } 703 #ifndef ZSTD_NOCOMPRESS 704 /* compression Level */ 705 if ((*argument>='0') && (*argument<='9')) { 706 dictCLevel = cLevel = readU32FromChar(&argument); 707 continue; 708 } 709 #endif 710 711 switch(argument[0]) 712 { 713 /* Display help */ 714 case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0); /* Version Only */ 715 case 'H': 716 case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); 717 718 /* Compress */ 719 case 'z': operation=zom_compress; argument++; break; 720 721 /* Decoding */ 722 case 'd': 723 #ifndef ZSTD_NOBENCH 724 benchParams.mode = BMK_decodeOnly; 725 if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ 726 #endif 727 operation=zom_decompress; argument++; break; 728 729 /* Force stdout, even if stdout==console */ 730 case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break; 731 732 /* Use file content as dictionary */ 733 case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; 734 735 /* Overwrite */ 736 case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break; 737 738 /* Verbose mode */ 739 case 'v': g_displayLevel++; argument++; break; 740 741 /* Quiet mode */ 742 case 'q': g_displayLevel--; argument++; break; 743 744 /* keep source file (default) */ 745 case 'k': FIO_setRemoveSrcFile(0); argument++; break; 746 747 /* Checksum */ 748 case 'C': FIO_setChecksumFlag(2); argument++; break; 749 750 /* test compressed file */ 751 case 't': operation=zom_test; argument++; break; 752 753 /* destination file name */ 754 case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; 755 756 /* limit decompression memory */ 757 case 'M': 758 argument++; 759 memLimit = readU32FromChar(&argument); 760 break; 761 case 'l': operation=zom_list; argument++; break; 762 #ifdef UTIL_HAS_CREATEFILELIST 763 /* recursive */ 764 case 'r': recursive=1; argument++; break; 765 #endif 766 767 #ifndef ZSTD_NOBENCH 768 /* Benchmark */ 769 case 'b': 770 operation=zom_bench; 771 argument++; 772 break; 773 774 /* range bench (benchmark only) */ 775 case 'e': 776 /* compression Level */ 777 argument++; 778 cLevelLast = readU32FromChar(&argument); 779 break; 780 781 /* Modify Nb Iterations (benchmark only) */ 782 case 'i': 783 argument++; 784 bench_nbSeconds = readU32FromChar(&argument); 785 break; 786 787 /* cut input into blocks (benchmark only) */ 788 case 'B': 789 argument++; 790 blockSize = readU32FromChar(&argument); 791 break; 792 793 /* benchmark files separately (hidden option) */ 794 case 'S': 795 argument++; 796 separateFiles = 1; 797 break; 798 799 #endif /* ZSTD_NOBENCH */ 800 801 /* nb of threads (hidden option) */ 802 case 'T': 803 argument++; 804 nbWorkers = readU32FromChar(&argument); 805 break; 806 807 /* Dictionary Selection level */ 808 case 's': 809 argument++; 810 dictSelect = readU32FromChar(&argument); 811 break; 812 813 /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ 814 case 'p': argument++; 815 #ifndef ZSTD_NOBENCH 816 if ((*argument>='0') && (*argument<='9')) { 817 benchParams.additionalParam = (int)readU32FromChar(&argument); 818 } else 819 #endif 820 main_pause=1; 821 break; 822 823 /* Select compressibility of synthetic sample */ 824 case 'P': 825 { argument++; 826 compressibility = (double)readU32FromChar(&argument) / 100; 827 } 828 break; 829 830 /* unknown command */ 831 default : CLEAN_RETURN(badusage(programName)); 832 } 833 } 834 continue; 835 } /* if (argument[0]=='-') */ 836 837 if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */ 838 nextArgumentIsMaxDict = 0; 839 lastCommand = 0; 840 maxDictSize = readU32FromChar(&argument); 841 continue; 842 } 843 844 if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */ 845 nextArgumentIsDictID = 0; 846 lastCommand = 0; 847 dictID = readU32FromChar(&argument); 848 continue; 849 } 850 851 } /* if (nextArgumentIsAFile==0) */ 852 853 if (nextEntryIsDictionary) { 854 nextEntryIsDictionary = 0; 855 lastCommand = 0; 856 dictFileName = argument; 857 continue; 858 } 859 860 if (nextArgumentIsOutFileName) { 861 nextArgumentIsOutFileName = 0; 862 lastCommand = 0; 863 outFileName = argument; 864 if (!strcmp(outFileName, "-")) outFileName = stdoutmark; 865 continue; 866 } 867 868 /* add filename to list */ 869 filenameTable[filenameIdx++] = argument; 870 } 871 872 if (lastCommand) { /* forgotten argument */ 873 DISPLAY("error : command must be followed by argument \n"); 874 CLEAN_RETURN(1); 875 } 876 877 /* Welcome message (if verbose) */ 878 DISPLAYLEVEL(3, WELCOME_MESSAGE); 879 880 #ifdef ZSTD_MULTITHREAD 881 if ((nbWorkers==0) && (!singleThread)) { 882 /* automatically set # workers based on # of reported cpus */ 883 nbWorkers = UTIL_countPhysicalCores(); 884 DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); 885 } 886 #else 887 (void)singleThread; (void)nbWorkers; 888 #endif 889 890 #ifdef UTIL_HAS_CREATEFILELIST 891 g_utilDisplayLevel = g_displayLevel; 892 if (!followLinks) { 893 unsigned u; 894 for (u=0, fileNamesNb=0; u<filenameIdx; u++) { 895 if (UTIL_isLink(filenameTable[u])) { 896 DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]); 897 } else { 898 filenameTable[fileNamesNb++] = filenameTable[u]; 899 } 900 } 901 filenameIdx = fileNamesNb; 902 } 903 if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ 904 extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); 905 if (extendedFileList) { 906 unsigned u; 907 for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]); 908 free((void*)filenameTable); 909 filenameTable = extendedFileList; 910 filenameIdx = fileNamesNb; 911 } 912 } 913 #else 914 (void)followLinks; 915 #endif 916 917 if (operation == zom_list) { 918 #ifndef ZSTD_NODECOMPRESS 919 int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel); 920 CLEAN_RETURN(ret); 921 #else 922 DISPLAY("file information is not supported \n"); 923 CLEAN_RETURN(1); 924 #endif 925 } 926 927 /* Check if benchmark is selected */ 928 if (operation==zom_bench) { 929 #ifndef ZSTD_NOBENCH 930 benchParams.blockSize = blockSize; 931 benchParams.nbWorkers = nbWorkers; 932 benchParams.realTime = setRealTimePrio; 933 benchParams.nbSeconds = bench_nbSeconds; 934 benchParams.ldmFlag = ldmFlag; 935 benchParams.ldmMinMatch = g_ldmMinMatch; 936 benchParams.ldmHashLog = g_ldmHashLog; 937 if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { 938 benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; 939 } 940 if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { 941 benchParams.ldmHashEveryLog = g_ldmHashEveryLog; 942 } 943 944 if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); 945 if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); 946 if (cLevelLast < cLevel) cLevelLast = cLevel; 947 if (cLevelLast > cLevel) 948 DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); 949 if(filenameIdx) { 950 if(separateFiles) { 951 unsigned i; 952 for(i = 0; i < filenameIdx; i++) { 953 int c; 954 DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]); 955 for(c = cLevel; c <= cLevelLast; c++) { 956 BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); 957 } 958 } 959 } else { 960 for(; cLevel <= cLevelLast; cLevel++) { 961 BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); 962 } 963 } 964 } else { 965 for(; cLevel <= cLevelLast; cLevel++) { 966 BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); 967 } 968 } 969 970 #else 971 (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; 972 #endif 973 goto _end; 974 } 975 976 /* Check if dictionary builder is selected */ 977 if (operation==zom_train) { 978 #ifndef ZSTD_NODICT 979 ZDICT_params_t zParams; 980 zParams.compressionLevel = dictCLevel; 981 zParams.notificationLevel = g_displayLevel; 982 zParams.dictID = dictID; 983 if (dict == cover) { 984 int const optimize = !coverParams.k || !coverParams.d; 985 coverParams.nbThreads = nbWorkers; 986 coverParams.zParams = zParams; 987 operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); 988 } else if (dict == fastCover) { 989 int const optimize = !fastCoverParams.k || !fastCoverParams.d; 990 fastCoverParams.nbThreads = nbWorkers; 991 fastCoverParams.zParams = zParams; 992 operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); 993 } else { 994 ZDICT_legacy_params_t dictParams; 995 memset(&dictParams, 0, sizeof(dictParams)); 996 dictParams.selectivityLevel = dictSelect; 997 dictParams.zParams = zParams; 998 operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0); 999 } 1000 #else 1001 (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */ 1002 DISPLAYLEVEL(1, "training mode not available \n"); 1003 operationResult = 1; 1004 #endif 1005 goto _end; 1006 } 1007 1008 #ifndef ZSTD_NODECOMPRESS 1009 if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ 1010 #endif 1011 1012 /* No input filename ==> use stdin and stdout */ 1013 filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */ 1014 if (!strcmp(filenameTable[0], stdinmark) && !outFileName) 1015 outFileName = stdoutmark; /* when input is stdin, default output is stdout */ 1016 1017 /* Check if input/output defined as console; trigger an error in this case */ 1018 if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) 1019 CLEAN_RETURN(badusage(programName)); 1020 if ( outFileName && !strcmp(outFileName, stdoutmark) 1021 && IS_CONSOLE(stdout) 1022 && !strcmp(filenameTable[0], stdinmark) 1023 && !forceStdout 1024 && operation!=zom_decompress ) 1025 CLEAN_RETURN(badusage(programName)); 1026 1027 #ifndef ZSTD_NOCOMPRESS 1028 /* check compression level limits */ 1029 { int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX; 1030 if (cLevel > maxCLevel) { 1031 DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel); 1032 cLevel = maxCLevel; 1033 } } 1034 #endif 1035 1036 /* No status message in pipe mode (stdin - stdout) or multi-files mode */ 1037 if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; 1038 if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1; 1039 1040 /* IO Stream/File */ 1041 FIO_setNotificationLevel(g_displayLevel); 1042 if (operation==zom_compress) { 1043 #ifndef ZSTD_NOCOMPRESS 1044 FIO_setNbWorkers(nbWorkers); 1045 FIO_setBlockSize((U32)blockSize); 1046 if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); 1047 FIO_setLdmFlag(ldmFlag); 1048 FIO_setLdmHashLog(g_ldmHashLog); 1049 FIO_setLdmMinMatch(g_ldmMinMatch); 1050 if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); 1051 if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog); 1052 FIO_setAdaptiveMode(adapt); 1053 FIO_setAdaptMin(adaptMin); 1054 FIO_setAdaptMax(adaptMax); 1055 if (adaptMin > cLevel) cLevel = adaptMin; 1056 if (adaptMax < cLevel) cLevel = adaptMax; 1057 1058 if ((filenameIdx==1) && outFileName) 1059 operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); 1060 else 1061 operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); 1062 #else 1063 (void)suffix; (void)adapt; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */ 1064 DISPLAY("Compression not supported \n"); 1065 #endif 1066 } else { /* decompression or test */ 1067 #ifndef ZSTD_NODECOMPRESS 1068 if (memLimit == 0) { 1069 if (compressionParams.windowLog == 0) 1070 memLimit = (U32)1 << g_defaultMaxWindowLog; 1071 else { 1072 memLimit = (U32)1 << (compressionParams.windowLog & 31); 1073 } 1074 } 1075 FIO_setMemLimit(memLimit); 1076 if (filenameIdx==1 && outFileName) 1077 operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); 1078 else 1079 operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName); 1080 #else 1081 DISPLAY("Decompression not supported \n"); 1082 #endif 1083 } 1084 1085 _end: 1086 if (main_pause) waitEnter(); 1087 #ifdef UTIL_HAS_CREATEFILELIST 1088 if (extendedFileList) 1089 UTIL_freeFileList(extendedFileList, fileNamesBuf); 1090 else 1091 #endif 1092 free((void*)filenameTable); 1093 return operationResult; 1094 } 1095