10c16b537SWarner Losh /* 2*37f1f268SConrad Meyer * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 30c16b537SWarner Losh * All rights reserved. 40c16b537SWarner Losh * 50c16b537SWarner Losh * This source code is licensed under both the BSD-style license (found in the 60c16b537SWarner Losh * LICENSE file in the root directory of this source tree) and the GPLv2 (found 70c16b537SWarner Losh * in the COPYING file in the root directory of this source tree). 80c16b537SWarner Losh * You may select, at your option, one of the above-listed licenses. 90c16b537SWarner Losh */ 100c16b537SWarner Losh #if defined (__cplusplus) 110c16b537SWarner Losh extern "C" { 120c16b537SWarner Losh #endif 130c16b537SWarner Losh 140c16b537SWarner Losh #ifndef ZSTD_H_235446 150c16b537SWarner Losh #define ZSTD_H_235446 160c16b537SWarner Losh 170c16b537SWarner Losh /* ====== Dependency ======*/ 189cbefe25SConrad Meyer #include <limits.h> /* INT_MAX */ 190c16b537SWarner Losh #include <stddef.h> /* size_t */ 200c16b537SWarner Losh 210c16b537SWarner Losh 220c16b537SWarner Losh /* ===== ZSTDLIB_API : control library symbols visibility ===== */ 230c16b537SWarner Losh #ifndef ZSTDLIB_VISIBILITY 240c16b537SWarner Losh # if defined(__GNUC__) && (__GNUC__ >= 4) 250c16b537SWarner Losh # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) 260c16b537SWarner Losh # else 270c16b537SWarner Losh # define ZSTDLIB_VISIBILITY 280c16b537SWarner Losh # endif 290c16b537SWarner Losh #endif 300c16b537SWarner Losh #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) 310c16b537SWarner Losh # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY 320c16b537SWarner Losh #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) 330c16b537SWarner Losh # define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 340c16b537SWarner Losh #else 350c16b537SWarner Losh # define ZSTDLIB_API ZSTDLIB_VISIBILITY 360c16b537SWarner Losh #endif 370c16b537SWarner Losh 380c16b537SWarner Losh 390f743729SConrad Meyer /******************************************************************************* 400c16b537SWarner Losh Introduction 410c16b537SWarner Losh 420f743729SConrad Meyer zstd, short for Zstandard, is a fast lossless compression algorithm, targeting 430f743729SConrad Meyer real-time compression scenarios at zlib-level and better compression ratios. 440f743729SConrad Meyer The zstd compression library provides in-memory compression and decompression 450f743729SConrad Meyer functions. 460f743729SConrad Meyer 470f743729SConrad Meyer The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), 480f743729SConrad Meyer which is currently 22. Levels >= 20, labeled `--ultra`, should be used with 490f743729SConrad Meyer caution, as they require more memory. The library also offers negative 500f743729SConrad Meyer compression levels, which extend the range of speed vs. ratio preferences. 510f743729SConrad Meyer The lower the level, the faster the speed (at the cost of compression). 520f743729SConrad Meyer 530c16b537SWarner Losh Compression can be done in: 540c16b537SWarner Losh - a single step (described as Simple API) 5519fcbaf1SConrad Meyer - a single step, reusing a context (described as Explicit context) 560c16b537SWarner Losh - unbounded multiple steps (described as Streaming compression) 570c16b537SWarner Losh 580f743729SConrad Meyer The compression ratio achievable on small data can be highly improved using 590f743729SConrad Meyer a dictionary. Dictionary compression can be performed in: 600f743729SConrad Meyer - a single step (described as Simple dictionary API) 610f743729SConrad Meyer - a single step, reusing a dictionary (described as Bulk-processing 620f743729SConrad Meyer dictionary API) 630f743729SConrad Meyer 640f743729SConrad Meyer Advanced experimental functions can be accessed using 650f743729SConrad Meyer `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. 660f743729SConrad Meyer 670f743729SConrad Meyer Advanced experimental APIs should never be used with a dynamically-linked 680f743729SConrad Meyer library. They are not "stable"; their definitions or signatures may change in 690f743729SConrad Meyer the future. Only static linking is allowed. 700f743729SConrad Meyer *******************************************************************************/ 710c16b537SWarner Losh 720c16b537SWarner Losh /*------ Version ------*/ 730c16b537SWarner Losh #define ZSTD_VERSION_MAJOR 1 742b9c00cbSConrad Meyer #define ZSTD_VERSION_MINOR 4 75*37f1f268SConrad Meyer #define ZSTD_VERSION_RELEASE 5 760c16b537SWarner Losh 770c16b537SWarner Losh #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) 78a0483764SConrad Meyer ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ 790c16b537SWarner Losh 800c16b537SWarner Losh #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE 810c16b537SWarner Losh #define ZSTD_QUOTE(str) #str 820c16b537SWarner Losh #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) 830c16b537SWarner Losh #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) 84a0483764SConrad Meyer ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ 850c16b537SWarner Losh 864d3f1eafSConrad Meyer /* ************************************* 870f743729SConrad Meyer * Default constant 880f743729SConrad Meyer ***************************************/ 890f743729SConrad Meyer #ifndef ZSTD_CLEVEL_DEFAULT 900f743729SConrad Meyer # define ZSTD_CLEVEL_DEFAULT 3 910f743729SConrad Meyer #endif 920c16b537SWarner Losh 934d3f1eafSConrad Meyer /* ************************************* 942b9c00cbSConrad Meyer * Constants 952b9c00cbSConrad Meyer ***************************************/ 962b9c00cbSConrad Meyer 972b9c00cbSConrad Meyer /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ 982b9c00cbSConrad Meyer #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ 992b9c00cbSConrad Meyer #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ 1002b9c00cbSConrad Meyer #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ 1012b9c00cbSConrad Meyer #define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 1022b9c00cbSConrad Meyer 1032b9c00cbSConrad Meyer #define ZSTD_BLOCKSIZELOG_MAX 17 1042b9c00cbSConrad Meyer #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) 1052b9c00cbSConrad Meyer 1062b9c00cbSConrad Meyer 1072b9c00cbSConrad Meyer 1082b9c00cbSConrad Meyer /*************************************** 1090c16b537SWarner Losh * Simple API 1100c16b537SWarner Losh ***************************************/ 1110c16b537SWarner Losh /*! ZSTD_compress() : 1120c16b537SWarner Losh * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. 1130c16b537SWarner Losh * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. 1140c16b537SWarner Losh * @return : compressed size written into `dst` (<= `dstCapacity), 1150c16b537SWarner Losh * or an error code if it fails (which can be tested using ZSTD_isError()). */ 1160c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, 1170c16b537SWarner Losh const void* src, size_t srcSize, 1180c16b537SWarner Losh int compressionLevel); 1190c16b537SWarner Losh 1200c16b537SWarner Losh /*! ZSTD_decompress() : 1210c16b537SWarner Losh * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. 1220c16b537SWarner Losh * `dstCapacity` is an upper bound of originalSize to regenerate. 1230c16b537SWarner Losh * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. 1240c16b537SWarner Losh * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), 1250c16b537SWarner Losh * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ 1260c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, 1270c16b537SWarner Losh const void* src, size_t compressedSize); 1280c16b537SWarner Losh 129a0483764SConrad Meyer /*! ZSTD_getFrameContentSize() : requires v1.3.0+ 1300c16b537SWarner Losh * `src` should point to the start of a ZSTD encoded frame. 1310c16b537SWarner Losh * `srcSize` must be at least as large as the frame header. 1320c16b537SWarner Losh * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. 1330f743729SConrad Meyer * @return : - decompressed size of `src` frame content, if known 1340c16b537SWarner Losh * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined 1350c16b537SWarner Losh * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) 1360c16b537SWarner Losh * note 1 : a 0 return value means the frame is valid but "empty". 1370c16b537SWarner Losh * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. 1380c16b537SWarner Losh * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. 1390c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 1400c16b537SWarner Losh * Optionally, application can rely on some implicit limit, 1410c16b537SWarner Losh * as ZSTD_decompress() only needs an upper bound of decompressed size. 1420c16b537SWarner Losh * (For example, data could be necessarily cut into blocks <= 16 KB). 1430f743729SConrad Meyer * note 3 : decompressed size is always present when compression is completed using single-pass functions, 1440f743729SConrad Meyer * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). 1450c16b537SWarner Losh * note 4 : decompressed size can be very large (64-bits value), 1460c16b537SWarner Losh * potentially larger than what local system can handle as a single memory segment. 1470c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 1480c16b537SWarner Losh * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. 1490c16b537SWarner Losh * Always ensure return value fits within application's authorized limits. 1500c16b537SWarner Losh * Each application can set its own limits. 1510c16b537SWarner Losh * note 6 : This function replaces ZSTD_getDecompressedSize() */ 1520c16b537SWarner Losh #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) 1530c16b537SWarner Losh #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) 1540c16b537SWarner Losh ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); 1550c16b537SWarner Losh 1560c16b537SWarner Losh /*! ZSTD_getDecompressedSize() : 1570c16b537SWarner Losh * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). 15819fcbaf1SConrad Meyer * Both functions work the same way, but ZSTD_getDecompressedSize() blends 15919fcbaf1SConrad Meyer * "empty", "unknown" and "error" results to the same return value (0), 16019fcbaf1SConrad Meyer * while ZSTD_getFrameContentSize() gives them separate return values. 1610f743729SConrad Meyer * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ 1620c16b537SWarner Losh ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); 1630c16b537SWarner Losh 1642b9c00cbSConrad Meyer /*! ZSTD_findFrameCompressedSize() : 1652b9c00cbSConrad Meyer * `src` should point to the start of a ZSTD frame or skippable frame. 1662b9c00cbSConrad Meyer * `srcSize` must be >= first frame size 1672b9c00cbSConrad Meyer * @return : the compressed size of the first frame starting at `src`, 1682b9c00cbSConrad Meyer * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, 1692b9c00cbSConrad Meyer * or an error code if input is invalid */ 1702b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); 1712b9c00cbSConrad Meyer 1720c16b537SWarner Losh 1730c16b537SWarner Losh /*====== Helper functions ======*/ 174052d3c12SConrad Meyer #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ 17519fcbaf1SConrad Meyer ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ 1760c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ 1770c16b537SWarner Losh ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ 1782b9c00cbSConrad Meyer ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ 1790c16b537SWarner Losh ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ 1800c16b537SWarner Losh 1810c16b537SWarner Losh 1820c16b537SWarner Losh /*************************************** 18319fcbaf1SConrad Meyer * Explicit context 1840c16b537SWarner Losh ***************************************/ 1850c16b537SWarner Losh /*= Compression context 1860c16b537SWarner Losh * When compressing many times, 1874d3f1eafSConrad Meyer * it is recommended to allocate a context just once, 1884d3f1eafSConrad Meyer * and re-use it for each successive compression operation. 1890c16b537SWarner Losh * This will make workload friendlier for system's memory. 1904d3f1eafSConrad Meyer * Note : re-using context is just a speed / resource optimization. 1914d3f1eafSConrad Meyer * It doesn't change the compression ratio, which remains identical. 1924d3f1eafSConrad Meyer * Note 2 : In multi-threaded environments, 1934d3f1eafSConrad Meyer * use one different context per thread for parallel execution. 1944d3f1eafSConrad Meyer */ 1950c16b537SWarner Losh typedef struct ZSTD_CCtx_s ZSTD_CCtx; 1960c16b537SWarner Losh ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); 1970c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); 1980c16b537SWarner Losh 1990c16b537SWarner Losh /*! ZSTD_compressCCtx() : 2009cbefe25SConrad Meyer * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. 2019cbefe25SConrad Meyer * Important : in order to behave similarly to `ZSTD_compress()`, 2029cbefe25SConrad Meyer * this function compresses at requested compression level, 2039cbefe25SConrad Meyer * __ignoring any other parameter__ . 2049cbefe25SConrad Meyer * If any advanced parameter was set using the advanced API, 2059cbefe25SConrad Meyer * they will all be reset. Only `compressionLevel` remains. 2069cbefe25SConrad Meyer */ 207a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, 2080c16b537SWarner Losh void* dst, size_t dstCapacity, 2090c16b537SWarner Losh const void* src, size_t srcSize, 2100c16b537SWarner Losh int compressionLevel); 2110c16b537SWarner Losh 2120c16b537SWarner Losh /*= Decompression context 2130c16b537SWarner Losh * When decompressing many times, 2140c16b537SWarner Losh * it is recommended to allocate a context only once, 2150c16b537SWarner Losh * and re-use it for each successive compression operation. 2160c16b537SWarner Losh * This will make workload friendlier for system's memory. 2170c16b537SWarner Losh * Use one context per thread for parallel execution. */ 2180c16b537SWarner Losh typedef struct ZSTD_DCtx_s ZSTD_DCtx; 2190c16b537SWarner Losh ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); 2200c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); 2210c16b537SWarner Losh 2220c16b537SWarner Losh /*! ZSTD_decompressDCtx() : 223a0483764SConrad Meyer * Same as ZSTD_decompress(), 224a0483764SConrad Meyer * requires an allocated ZSTD_DCtx. 225a0483764SConrad Meyer * Compatible with sticky parameters. 226a0483764SConrad Meyer */ 227a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, 2280c16b537SWarner Losh void* dst, size_t dstCapacity, 2290c16b537SWarner Losh const void* src, size_t srcSize); 2300c16b537SWarner Losh 2310c16b537SWarner Losh 232a0483764SConrad Meyer /*************************************** 233a0483764SConrad Meyer * Advanced compression API 234a0483764SConrad Meyer ***************************************/ 235a0483764SConrad Meyer 236a0483764SConrad Meyer /* API design : 237a0483764SConrad Meyer * Parameters are pushed one by one into an existing context, 238a0483764SConrad Meyer * using ZSTD_CCtx_set*() functions. 239a0483764SConrad Meyer * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. 240a0483764SConrad Meyer * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! 2419cbefe25SConrad Meyer * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . 242a0483764SConrad Meyer * 243a0483764SConrad Meyer * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). 244a0483764SConrad Meyer * 245a0483764SConrad Meyer * This API supercedes all other "advanced" API entry points in the experimental section. 246a0483764SConrad Meyer * In the future, we expect to remove from experimental API entry points which are redundant with this API. 247a0483764SConrad Meyer */ 248a0483764SConrad Meyer 249a0483764SConrad Meyer 250a0483764SConrad Meyer /* Compression strategies, listed from fastest to strongest */ 251a0483764SConrad Meyer typedef enum { ZSTD_fast=1, 252a0483764SConrad Meyer ZSTD_dfast=2, 253a0483764SConrad Meyer ZSTD_greedy=3, 254a0483764SConrad Meyer ZSTD_lazy=4, 255a0483764SConrad Meyer ZSTD_lazy2=5, 256a0483764SConrad Meyer ZSTD_btlazy2=6, 257a0483764SConrad Meyer ZSTD_btopt=7, 258a0483764SConrad Meyer ZSTD_btultra=8, 259a0483764SConrad Meyer ZSTD_btultra2=9 260a0483764SConrad Meyer /* note : new strategies _might_ be added in the future. 261a0483764SConrad Meyer Only the order (from fast to strong) is guaranteed */ 262a0483764SConrad Meyer } ZSTD_strategy; 263a0483764SConrad Meyer 264a0483764SConrad Meyer 265a0483764SConrad Meyer typedef enum { 266a0483764SConrad Meyer 2672b9c00cbSConrad Meyer /* compression parameters 2682b9c00cbSConrad Meyer * Note: When compressing with a ZSTD_CDict these parameters are superseded 2699cbefe25SConrad Meyer * by the parameters used to construct the ZSTD_CDict. 2709cbefe25SConrad Meyer * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ 2719cbefe25SConrad Meyer ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. 2729cbefe25SConrad Meyer * Note that exact compression parameters are dynamically determined, 2739cbefe25SConrad Meyer * depending on both compression level and srcSize (when known). 274a0483764SConrad Meyer * Default level is ZSTD_CLEVEL_DEFAULT==3. 275a0483764SConrad Meyer * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. 276a0483764SConrad Meyer * Note 1 : it's possible to pass a negative compression level. 277*37f1f268SConrad Meyer * Note 2 : setting a level does not automatically set all other compression parameters 278*37f1f268SConrad Meyer * to default. Setting this will however eventually dynamically impact the compression 279*37f1f268SConrad Meyer * parameters which have not been manually set. The manually set 280*37f1f268SConrad Meyer * ones will 'stick'. */ 2819cbefe25SConrad Meyer /* Advanced compression parameters : 2829cbefe25SConrad Meyer * It's possible to pin down compression parameters to some specific values. 2839cbefe25SConrad Meyer * In which case, these values are no longer dynamically selected by the compressor */ 284a0483764SConrad Meyer ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. 2859cbefe25SConrad Meyer * This will set a memory budget for streaming decompression, 2869cbefe25SConrad Meyer * with larger values requiring more memory 2879cbefe25SConrad Meyer * and typically compressing more. 288a0483764SConrad Meyer * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. 289a0483764SConrad Meyer * Special: value 0 means "use default windowLog". 290a0483764SConrad Meyer * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT 2919cbefe25SConrad Meyer * requires explicitly allowing such size at streaming decompression stage. */ 292a0483764SConrad Meyer ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. 293a0483764SConrad Meyer * Resulting memory usage is (1 << (hashLog+2)). 294a0483764SConrad Meyer * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. 295a0483764SConrad Meyer * Larger tables improve compression ratio of strategies <= dFast, 296a0483764SConrad Meyer * and improve speed of strategies > dFast. 297a0483764SConrad Meyer * Special: value 0 means "use default hashLog". */ 298a0483764SConrad Meyer ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. 299a0483764SConrad Meyer * Resulting memory usage is (1 << (chainLog+2)). 300a0483764SConrad Meyer * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. 301a0483764SConrad Meyer * Larger tables result in better and slower compression. 3029cbefe25SConrad Meyer * This parameter is useless for "fast" strategy. 303a0483764SConrad Meyer * It's still useful when using "dfast" strategy, 304a0483764SConrad Meyer * in which case it defines a secondary probe table. 305a0483764SConrad Meyer * Special: value 0 means "use default chainLog". */ 306a0483764SConrad Meyer ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. 307a0483764SConrad Meyer * More attempts result in better and slower compression. 3089cbefe25SConrad Meyer * This parameter is useless for "fast" and "dFast" strategies. 309a0483764SConrad Meyer * Special: value 0 means "use default searchLog". */ 310a0483764SConrad Meyer ZSTD_c_minMatch=105, /* Minimum size of searched matches. 311a0483764SConrad Meyer * Note that Zstandard can still find matches of smaller size, 312a0483764SConrad Meyer * it just tweaks its search algorithm to look for this size and larger. 313a0483764SConrad Meyer * Larger values increase compression and decompression speed, but decrease ratio. 314a0483764SConrad Meyer * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. 315a0483764SConrad Meyer * Note that currently, for all strategies < btopt, effective minimum is 4. 316a0483764SConrad Meyer * , for all strategies > fast, effective maximum is 6. 317a0483764SConrad Meyer * Special: value 0 means "use default minMatchLength". */ 318a0483764SConrad Meyer ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. 319a0483764SConrad Meyer * For strategies btopt, btultra & btultra2: 320a0483764SConrad Meyer * Length of Match considered "good enough" to stop search. 321a0483764SConrad Meyer * Larger values make compression stronger, and slower. 322a0483764SConrad Meyer * For strategy fast: 323a0483764SConrad Meyer * Distance between match sampling. 324a0483764SConrad Meyer * Larger values make compression faster, and weaker. 325a0483764SConrad Meyer * Special: value 0 means "use default targetLength". */ 326a0483764SConrad Meyer ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. 327a0483764SConrad Meyer * The higher the value of selected strategy, the more complex it is, 328a0483764SConrad Meyer * resulting in stronger and slower compression. 329a0483764SConrad Meyer * Special: value 0 means "use default strategy". */ 330a0483764SConrad Meyer 331a0483764SConrad Meyer /* LDM mode parameters */ 332a0483764SConrad Meyer ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. 333a0483764SConrad Meyer * This parameter is designed to improve compression ratio 334a0483764SConrad Meyer * for large inputs, by finding large matches at long distance. 335a0483764SConrad Meyer * It increases memory usage and window size. 336a0483764SConrad Meyer * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB 337a0483764SConrad Meyer * except when expressly set to a different value. */ 338a0483764SConrad Meyer ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. 339a0483764SConrad Meyer * Larger values increase memory usage and compression ratio, 340a0483764SConrad Meyer * but decrease compression speed. 341a0483764SConrad Meyer * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX 342a0483764SConrad Meyer * default: windowlog - 7. 343a0483764SConrad Meyer * Special: value 0 means "automatically determine hashlog". */ 344a0483764SConrad Meyer ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. 345a0483764SConrad Meyer * Larger/too small values usually decrease compression ratio. 346a0483764SConrad Meyer * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. 347a0483764SConrad Meyer * Special: value 0 means "use default value" (default: 64). */ 348a0483764SConrad Meyer ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. 349a0483764SConrad Meyer * Larger values improve collision resolution but decrease compression speed. 350a0483764SConrad Meyer * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. 351a0483764SConrad Meyer * Special: value 0 means "use default value" (default: 3). */ 352a0483764SConrad Meyer ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. 353a0483764SConrad Meyer * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). 354a0483764SConrad Meyer * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. 355a0483764SConrad Meyer * Larger values improve compression speed. 356a0483764SConrad Meyer * Deviating far from default value will likely result in a compression ratio decrease. 357a0483764SConrad Meyer * Special: value 0 means "automatically determine hashRateLog". */ 358a0483764SConrad Meyer 359a0483764SConrad Meyer /* frame parameters */ 360a0483764SConrad Meyer ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) 361a0483764SConrad Meyer * Content size must be known at the beginning of compression. 362a0483764SConrad Meyer * This is automatically the case when using ZSTD_compress2(), 3639cbefe25SConrad Meyer * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ 364a0483764SConrad Meyer ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ 365a0483764SConrad Meyer ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ 366a0483764SConrad Meyer 367a0483764SConrad Meyer /* multi-threading parameters */ 368a0483764SConrad Meyer /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). 369a0483764SConrad Meyer * They return an error otherwise. */ 370a0483764SConrad Meyer ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. 371a0483764SConrad Meyer * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : 372a0483764SConrad Meyer * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, 373a0483764SConrad Meyer * while compression work is performed in parallel, within worker threads. 374a0483764SConrad Meyer * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : 375a0483764SConrad Meyer * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). 376a0483764SConrad Meyer * More workers improve speed, but also increase memory usage. 377a0483764SConrad Meyer * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ 378a0483764SConrad Meyer ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. 379a0483764SConrad Meyer * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. 380a0483764SConrad Meyer * 0 means default, which is dynamically determined based on compression parameters. 381a0483764SConrad Meyer * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. 3829cbefe25SConrad Meyer * The minimum size is automatically and transparently enforced. */ 383a0483764SConrad Meyer ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. 384a0483764SConrad Meyer * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. 385a0483764SConrad Meyer * It helps preserve compression ratio, while each job is compressed in parallel. 386a0483764SConrad Meyer * This value is enforced only when nbWorkers >= 1. 387a0483764SConrad Meyer * Larger values increase compression ratio, but decrease speed. 388a0483764SConrad Meyer * Possible values range from 0 to 9 : 389a0483764SConrad Meyer * - 0 means "default" : value will be determined by the library, depending on strategy 390a0483764SConrad Meyer * - 1 means "no overlap" 391a0483764SConrad Meyer * - 9 means "full overlap", using a full window size. 392a0483764SConrad Meyer * Each intermediate rank increases/decreases load size by a factor 2 : 393a0483764SConrad Meyer * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default 394a0483764SConrad Meyer * default value varies between 6 and 9, depending on strategy */ 395a0483764SConrad Meyer 396a0483764SConrad Meyer /* note : additional experimental parameters are also available 397a0483764SConrad Meyer * within the experimental section of the API. 398a0483764SConrad Meyer * At the time of this writing, they include : 399a0483764SConrad Meyer * ZSTD_c_rsyncable 400a0483764SConrad Meyer * ZSTD_c_format 401a0483764SConrad Meyer * ZSTD_c_forceMaxWindow 402a0483764SConrad Meyer * ZSTD_c_forceAttachDict 4032b9c00cbSConrad Meyer * ZSTD_c_literalCompressionMode 4044d3f1eafSConrad Meyer * ZSTD_c_targetCBlockSize 4059cbefe25SConrad Meyer * ZSTD_c_srcSizeHint 406a0483764SConrad Meyer * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. 407a0483764SConrad Meyer * note : never ever use experimentalParam? names directly; 408a0483764SConrad Meyer * also, the enums values themselves are unstable and can still change. 409a0483764SConrad Meyer */ 410a0483764SConrad Meyer ZSTD_c_experimentalParam1=500, 411a0483764SConrad Meyer ZSTD_c_experimentalParam2=10, 412a0483764SConrad Meyer ZSTD_c_experimentalParam3=1000, 4132b9c00cbSConrad Meyer ZSTD_c_experimentalParam4=1001, 4142b9c00cbSConrad Meyer ZSTD_c_experimentalParam5=1002, 4154d3f1eafSConrad Meyer ZSTD_c_experimentalParam6=1003, 4169cbefe25SConrad Meyer ZSTD_c_experimentalParam7=1004 417a0483764SConrad Meyer } ZSTD_cParameter; 418a0483764SConrad Meyer 419a0483764SConrad Meyer typedef struct { 420a0483764SConrad Meyer size_t error; 421a0483764SConrad Meyer int lowerBound; 422a0483764SConrad Meyer int upperBound; 423a0483764SConrad Meyer } ZSTD_bounds; 424a0483764SConrad Meyer 425a0483764SConrad Meyer /*! ZSTD_cParam_getBounds() : 426a0483764SConrad Meyer * All parameters must belong to an interval with lower and upper bounds, 427a0483764SConrad Meyer * otherwise they will either trigger an error or be automatically clamped. 428a0483764SConrad Meyer * @return : a structure, ZSTD_bounds, which contains 429a0483764SConrad Meyer * - an error status field, which must be tested using ZSTD_isError() 430a0483764SConrad Meyer * - lower and upper bounds, both inclusive 431a0483764SConrad Meyer */ 432a0483764SConrad Meyer ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); 433a0483764SConrad Meyer 434a0483764SConrad Meyer /*! ZSTD_CCtx_setParameter() : 435a0483764SConrad Meyer * Set one compression parameter, selected by enum ZSTD_cParameter. 436a0483764SConrad Meyer * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). 437a0483764SConrad Meyer * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). 438a0483764SConrad Meyer * Setting a parameter is generally only possible during frame initialization (before starting compression). 439a0483764SConrad Meyer * Exception : when using multi-threading mode (nbWorkers >= 1), 440a0483764SConrad Meyer * the following parameters can be updated _during_ compression (within same frame): 441a0483764SConrad Meyer * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. 442a0483764SConrad Meyer * new parameters will be active for next job only (after a flush()). 443a0483764SConrad Meyer * @return : an error code (which can be tested using ZSTD_isError()). 444a0483764SConrad Meyer */ 445a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); 446a0483764SConrad Meyer 447a0483764SConrad Meyer /*! ZSTD_CCtx_setPledgedSrcSize() : 448a0483764SConrad Meyer * Total input data size to be compressed as a single frame. 449a0483764SConrad Meyer * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. 450a0483764SConrad Meyer * This value will also be controlled at end of frame, and trigger an error if not respected. 451a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 452a0483764SConrad Meyer * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. 453a0483764SConrad Meyer * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. 454a0483764SConrad Meyer * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. 455a0483764SConrad Meyer * Note 2 : pledgedSrcSize is only valid once, for the next frame. 456a0483764SConrad Meyer * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. 457a0483764SConrad Meyer * Note 3 : Whenever all input data is provided and consumed in a single round, 458a0483764SConrad Meyer * for example with ZSTD_compress2(), 459a0483764SConrad Meyer * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), 4602b9c00cbSConrad Meyer * this value is automatically overridden by srcSize instead. 461a0483764SConrad Meyer */ 462a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); 463a0483764SConrad Meyer 4642b9c00cbSConrad Meyer typedef enum { 4652b9c00cbSConrad Meyer ZSTD_reset_session_only = 1, 4662b9c00cbSConrad Meyer ZSTD_reset_parameters = 2, 4672b9c00cbSConrad Meyer ZSTD_reset_session_and_parameters = 3 4682b9c00cbSConrad Meyer } ZSTD_ResetDirective; 4692b9c00cbSConrad Meyer 4702b9c00cbSConrad Meyer /*! ZSTD_CCtx_reset() : 4712b9c00cbSConrad Meyer * There are 2 different things that can be reset, independently or jointly : 4722b9c00cbSConrad Meyer * - The session : will stop compressing current frame, and make CCtx ready to start a new one. 4732b9c00cbSConrad Meyer * Useful after an error, or to interrupt any ongoing compression. 4742b9c00cbSConrad Meyer * Any internal data not yet flushed is cancelled. 4752b9c00cbSConrad Meyer * Compression parameters and dictionary remain unchanged. 4762b9c00cbSConrad Meyer * They will be used to compress next frame. 4772b9c00cbSConrad Meyer * Resetting session never fails. 4782b9c00cbSConrad Meyer * - The parameters : changes all parameters back to "default". 4792b9c00cbSConrad Meyer * This removes any reference to any dictionary too. 4802b9c00cbSConrad Meyer * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) 4812b9c00cbSConrad Meyer * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) 4822b9c00cbSConrad Meyer * - Both : similar to resetting the session, followed by resetting parameters. 4832b9c00cbSConrad Meyer */ 4842b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); 4852b9c00cbSConrad Meyer 4862b9c00cbSConrad Meyer /*! ZSTD_compress2() : 4872b9c00cbSConrad Meyer * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. 4882b9c00cbSConrad Meyer * ZSTD_compress2() always starts a new frame. 4892b9c00cbSConrad Meyer * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. 4902b9c00cbSConrad Meyer * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() 4912b9c00cbSConrad Meyer * - The function is always blocking, returns when compression is completed. 4922b9c00cbSConrad Meyer * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. 4932b9c00cbSConrad Meyer * @return : compressed size written into `dst` (<= `dstCapacity), 4942b9c00cbSConrad Meyer * or an error code if it fails (which can be tested using ZSTD_isError()). 4952b9c00cbSConrad Meyer */ 4962b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, 4972b9c00cbSConrad Meyer void* dst, size_t dstCapacity, 4982b9c00cbSConrad Meyer const void* src, size_t srcSize); 4992b9c00cbSConrad Meyer 5002b9c00cbSConrad Meyer 5012b9c00cbSConrad Meyer /*************************************** 5022b9c00cbSConrad Meyer * Advanced decompression API 5032b9c00cbSConrad Meyer ***************************************/ 5042b9c00cbSConrad Meyer 5052b9c00cbSConrad Meyer /* The advanced API pushes parameters one by one into an existing DCtx context. 5062b9c00cbSConrad Meyer * Parameters are sticky, and remain valid for all following frames 5072b9c00cbSConrad Meyer * using the same DCtx context. 5082b9c00cbSConrad Meyer * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). 5092b9c00cbSConrad Meyer * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). 5102b9c00cbSConrad Meyer * Therefore, no new decompression function is necessary. 5112b9c00cbSConrad Meyer */ 5122b9c00cbSConrad Meyer 5132b9c00cbSConrad Meyer typedef enum { 5142b9c00cbSConrad Meyer 5152b9c00cbSConrad Meyer ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which 5162b9c00cbSConrad Meyer * the streaming API will refuse to allocate memory buffer 5172b9c00cbSConrad Meyer * in order to protect the host from unreasonable memory requirements. 5182b9c00cbSConrad Meyer * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. 5192b9c00cbSConrad Meyer * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). 5202b9c00cbSConrad Meyer * Special: value 0 means "use default maximum windowLog". */ 5212b9c00cbSConrad Meyer 5222b9c00cbSConrad Meyer /* note : additional experimental parameters are also available 5232b9c00cbSConrad Meyer * within the experimental section of the API. 5242b9c00cbSConrad Meyer * At the time of this writing, they include : 525*37f1f268SConrad Meyer * ZSTD_d_format 526*37f1f268SConrad Meyer * ZSTD_d_stableOutBuffer 5272b9c00cbSConrad Meyer * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. 5282b9c00cbSConrad Meyer * note : never ever use experimentalParam? names directly 5292b9c00cbSConrad Meyer */ 530*37f1f268SConrad Meyer ZSTD_d_experimentalParam1=1000, 531*37f1f268SConrad Meyer ZSTD_d_experimentalParam2=1001 5322b9c00cbSConrad Meyer 5332b9c00cbSConrad Meyer } ZSTD_dParameter; 5342b9c00cbSConrad Meyer 5352b9c00cbSConrad Meyer /*! ZSTD_dParam_getBounds() : 5362b9c00cbSConrad Meyer * All parameters must belong to an interval with lower and upper bounds, 5372b9c00cbSConrad Meyer * otherwise they will either trigger an error or be automatically clamped. 5382b9c00cbSConrad Meyer * @return : a structure, ZSTD_bounds, which contains 5392b9c00cbSConrad Meyer * - an error status field, which must be tested using ZSTD_isError() 5402b9c00cbSConrad Meyer * - both lower and upper bounds, inclusive 5412b9c00cbSConrad Meyer */ 5422b9c00cbSConrad Meyer ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); 5432b9c00cbSConrad Meyer 5442b9c00cbSConrad Meyer /*! ZSTD_DCtx_setParameter() : 5452b9c00cbSConrad Meyer * Set one compression parameter, selected by enum ZSTD_dParameter. 5462b9c00cbSConrad Meyer * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). 5472b9c00cbSConrad Meyer * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). 5482b9c00cbSConrad Meyer * Setting a parameter is only possible during frame initialization (before starting decompression). 5492b9c00cbSConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()). 5502b9c00cbSConrad Meyer */ 5512b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); 5522b9c00cbSConrad Meyer 5532b9c00cbSConrad Meyer /*! ZSTD_DCtx_reset() : 5542b9c00cbSConrad Meyer * Return a DCtx to clean state. 5552b9c00cbSConrad Meyer * Session and parameters can be reset jointly or separately. 5562b9c00cbSConrad Meyer * Parameters can only be reset when no active frame is being decompressed. 5572b9c00cbSConrad Meyer * @return : 0, or an error code, which can be tested with ZSTD_isError() 5582b9c00cbSConrad Meyer */ 5592b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); 5602b9c00cbSConrad Meyer 5612b9c00cbSConrad Meyer 5622b9c00cbSConrad Meyer /**************************** 5632b9c00cbSConrad Meyer * Streaming 5642b9c00cbSConrad Meyer ****************************/ 5652b9c00cbSConrad Meyer 5662b9c00cbSConrad Meyer typedef struct ZSTD_inBuffer_s { 5672b9c00cbSConrad Meyer const void* src; /**< start of input buffer */ 5682b9c00cbSConrad Meyer size_t size; /**< size of input buffer */ 5692b9c00cbSConrad Meyer size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ 5702b9c00cbSConrad Meyer } ZSTD_inBuffer; 5712b9c00cbSConrad Meyer 5722b9c00cbSConrad Meyer typedef struct ZSTD_outBuffer_s { 5732b9c00cbSConrad Meyer void* dst; /**< start of output buffer */ 5742b9c00cbSConrad Meyer size_t size; /**< size of output buffer */ 5752b9c00cbSConrad Meyer size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ 5762b9c00cbSConrad Meyer } ZSTD_outBuffer; 5772b9c00cbSConrad Meyer 5782b9c00cbSConrad Meyer 5792b9c00cbSConrad Meyer 5802b9c00cbSConrad Meyer /*-*********************************************************************** 5812b9c00cbSConrad Meyer * Streaming compression - HowTo 5822b9c00cbSConrad Meyer * 5832b9c00cbSConrad Meyer * A ZSTD_CStream object is required to track streaming operation. 5842b9c00cbSConrad Meyer * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. 5852b9c00cbSConrad Meyer * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. 5862b9c00cbSConrad Meyer * It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. 5872b9c00cbSConrad Meyer * 5882b9c00cbSConrad Meyer * For parallel execution, use one separate ZSTD_CStream per thread. 5892b9c00cbSConrad Meyer * 5902b9c00cbSConrad Meyer * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. 5912b9c00cbSConrad Meyer * 5922b9c00cbSConrad Meyer * Parameters are sticky : when starting a new compression on the same context, 5932b9c00cbSConrad Meyer * it will re-use the same sticky parameters as previous compression session. 5942b9c00cbSConrad Meyer * When in doubt, it's recommended to fully initialize the context before usage. 5952b9c00cbSConrad Meyer * Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), 5962b9c00cbSConrad Meyer * ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to 5972b9c00cbSConrad Meyer * set more specific parameters, the pledged source size, or load a dictionary. 5982b9c00cbSConrad Meyer * 5992b9c00cbSConrad Meyer * Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to 6002b9c00cbSConrad Meyer * consume input stream. The function will automatically update both `pos` 6012b9c00cbSConrad Meyer * fields within `input` and `output`. 6022b9c00cbSConrad Meyer * Note that the function may not consume the entire input, for example, because 6032b9c00cbSConrad Meyer * the output buffer is already full, in which case `input.pos < input.size`. 6042b9c00cbSConrad Meyer * The caller must check if input has been entirely consumed. 6052b9c00cbSConrad Meyer * If not, the caller must make some room to receive more compressed data, 6062b9c00cbSConrad Meyer * and then present again remaining input data. 6072b9c00cbSConrad Meyer * note: ZSTD_e_continue is guaranteed to make some forward progress when called, 6082b9c00cbSConrad Meyer * but doesn't guarantee maximal forward progress. This is especially relevant 6092b9c00cbSConrad Meyer * when compressing with multiple threads. The call won't block if it can 6102b9c00cbSConrad Meyer * consume some input, but if it can't it will wait for some, but not all, 6112b9c00cbSConrad Meyer * output to be flushed. 6122b9c00cbSConrad Meyer * @return : provides a minimum amount of data remaining to be flushed from internal buffers 6132b9c00cbSConrad Meyer * or an error code, which can be tested using ZSTD_isError(). 6142b9c00cbSConrad Meyer * 6152b9c00cbSConrad Meyer * At any moment, it's possible to flush whatever data might remain stuck within internal buffer, 6162b9c00cbSConrad Meyer * using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. 6172b9c00cbSConrad Meyer * Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). 6182b9c00cbSConrad Meyer * In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. 6192b9c00cbSConrad Meyer * You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the 6202b9c00cbSConrad Meyer * operation. 6212b9c00cbSConrad Meyer * note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will 6222b9c00cbSConrad Meyer * block until the flush is complete or the output buffer is full. 6232b9c00cbSConrad Meyer * @return : 0 if internal buffers are entirely flushed, 6242b9c00cbSConrad Meyer * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), 6252b9c00cbSConrad Meyer * or an error code, which can be tested using ZSTD_isError(). 6262b9c00cbSConrad Meyer * 6272b9c00cbSConrad Meyer * Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. 6282b9c00cbSConrad Meyer * It will perform a flush and write frame epilogue. 6292b9c00cbSConrad Meyer * The epilogue is required for decoders to consider a frame completed. 6302b9c00cbSConrad Meyer * flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. 6312b9c00cbSConrad Meyer * You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to 6322b9c00cbSConrad Meyer * start a new frame. 6332b9c00cbSConrad Meyer * note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will 6342b9c00cbSConrad Meyer * block until the flush is complete or the output buffer is full. 6352b9c00cbSConrad Meyer * @return : 0 if frame fully completed and fully flushed, 6362b9c00cbSConrad Meyer * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), 6372b9c00cbSConrad Meyer * or an error code, which can be tested using ZSTD_isError(). 6382b9c00cbSConrad Meyer * 6392b9c00cbSConrad Meyer * *******************************************************************/ 6402b9c00cbSConrad Meyer 6412b9c00cbSConrad Meyer typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ 6422b9c00cbSConrad Meyer /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ 6432b9c00cbSConrad Meyer /*===== ZSTD_CStream management functions =====*/ 6442b9c00cbSConrad Meyer ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); 6452b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); 6462b9c00cbSConrad Meyer 6472b9c00cbSConrad Meyer /*===== Streaming compression functions =====*/ 6482b9c00cbSConrad Meyer typedef enum { 6492b9c00cbSConrad Meyer ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ 6502b9c00cbSConrad Meyer ZSTD_e_flush=1, /* flush any data provided so far, 6512b9c00cbSConrad Meyer * it creates (at least) one new block, that can be decoded immediately on reception; 6522b9c00cbSConrad Meyer * frame will continue: any future data can still reference previously compressed data, improving compression. 6532b9c00cbSConrad Meyer * note : multithreaded compression will block to flush as much output as possible. */ 6542b9c00cbSConrad Meyer ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. 6552b9c00cbSConrad Meyer * note that frame is only closed after compressed data is fully flushed (return value == 0). 6562b9c00cbSConrad Meyer * After that point, any additional data starts a new frame. 6572b9c00cbSConrad Meyer * note : each frame is independent (does not reference any content from previous frame). 6582b9c00cbSConrad Meyer : note : multithreaded compression will block to flush as much output as possible. */ 6592b9c00cbSConrad Meyer } ZSTD_EndDirective; 6602b9c00cbSConrad Meyer 6612b9c00cbSConrad Meyer /*! ZSTD_compressStream2() : 6622b9c00cbSConrad Meyer * Behaves about the same as ZSTD_compressStream, with additional control on end directive. 6632b9c00cbSConrad Meyer * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() 6642b9c00cbSConrad Meyer * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) 6652b9c00cbSConrad Meyer * - output->pos must be <= dstCapacity, input->pos must be <= srcSize 6662b9c00cbSConrad Meyer * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. 6672b9c00cbSConrad Meyer * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. 6682b9c00cbSConrad Meyer * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, 6692b9c00cbSConrad Meyer * and then immediately returns, just indicating that there is some data remaining to be flushed. 6702b9c00cbSConrad Meyer * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. 6712b9c00cbSConrad Meyer * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. 6722b9c00cbSConrad Meyer * - @return provides a minimum amount of data remaining to be flushed from internal buffers 6732b9c00cbSConrad Meyer * or an error code, which can be tested using ZSTD_isError(). 6742b9c00cbSConrad Meyer * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. 6752b9c00cbSConrad Meyer * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. 6762b9c00cbSConrad Meyer * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. 6772b9c00cbSConrad Meyer * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), 6782b9c00cbSConrad Meyer * only ZSTD_e_end or ZSTD_e_flush operations are allowed. 6792b9c00cbSConrad Meyer * Before starting a new compression job, or changing compression parameters, 6802b9c00cbSConrad Meyer * it is required to fully flush internal buffers. 6812b9c00cbSConrad Meyer */ 6822b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, 6832b9c00cbSConrad Meyer ZSTD_outBuffer* output, 6842b9c00cbSConrad Meyer ZSTD_inBuffer* input, 6852b9c00cbSConrad Meyer ZSTD_EndDirective endOp); 6862b9c00cbSConrad Meyer 6872b9c00cbSConrad Meyer 6884d3f1eafSConrad Meyer /* These buffer sizes are softly recommended. 6894d3f1eafSConrad Meyer * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. 6904d3f1eafSConrad Meyer * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), 6914d3f1eafSConrad Meyer * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. 6924d3f1eafSConrad Meyer * 6934d3f1eafSConrad Meyer * However, note that these recommendations are from the perspective of a C caller program. 6944d3f1eafSConrad Meyer * If the streaming interface is invoked from some other language, 6954d3f1eafSConrad Meyer * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, 6964d3f1eafSConrad Meyer * a major performance rule is to reduce crossing such interface to an absolute minimum. 6974d3f1eafSConrad Meyer * It's not rare that performance ends being spent more into the interface, rather than compression itself. 6984d3f1eafSConrad Meyer * In which cases, prefer using large buffers, as large as practical, 6994d3f1eafSConrad Meyer * for both input and output, to reduce the nb of roundtrips. 7004d3f1eafSConrad Meyer */ 7014d3f1eafSConrad Meyer ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ 7024d3f1eafSConrad Meyer ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ 7034d3f1eafSConrad Meyer 7044d3f1eafSConrad Meyer 7054d3f1eafSConrad Meyer /* ***************************************************************************** 7064d3f1eafSConrad Meyer * This following is a legacy streaming API. 7074d3f1eafSConrad Meyer * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). 7084d3f1eafSConrad Meyer * It is redundant, but remains fully supported. 7092b9c00cbSConrad Meyer * Advanced parameters and dictionary compression can only be used through the 7102b9c00cbSConrad Meyer * new API. 7112b9c00cbSConrad Meyer ******************************************************************************/ 7122b9c00cbSConrad Meyer 7134d3f1eafSConrad Meyer /*! 7142b9c00cbSConrad Meyer * Equivalent to: 7152b9c00cbSConrad Meyer * 7162b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 7172b9c00cbSConrad Meyer * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) 7182b9c00cbSConrad Meyer * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); 7192b9c00cbSConrad Meyer */ 7202b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); 7214d3f1eafSConrad Meyer /*! 7222b9c00cbSConrad Meyer * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). 7232b9c00cbSConrad Meyer * NOTE: The return value is different. ZSTD_compressStream() returns a hint for 7242b9c00cbSConrad Meyer * the next read size (if non-zero and not an error). ZSTD_compressStream2() 7254d3f1eafSConrad Meyer * returns the minimum nb of bytes left to flush (if non-zero and not an error). 7262b9c00cbSConrad Meyer */ 7272b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); 7284d3f1eafSConrad Meyer /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ 7292b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); 7304d3f1eafSConrad Meyer /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ 7312b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); 7322b9c00cbSConrad Meyer 7332b9c00cbSConrad Meyer 7342b9c00cbSConrad Meyer /*-*************************************************************************** 7352b9c00cbSConrad Meyer * Streaming decompression - HowTo 7362b9c00cbSConrad Meyer * 7372b9c00cbSConrad Meyer * A ZSTD_DStream object is required to track streaming operations. 7382b9c00cbSConrad Meyer * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. 7392b9c00cbSConrad Meyer * ZSTD_DStream objects can be re-used multiple times. 7402b9c00cbSConrad Meyer * 7412b9c00cbSConrad Meyer * Use ZSTD_initDStream() to start a new decompression operation. 7422b9c00cbSConrad Meyer * @return : recommended first input size 7432b9c00cbSConrad Meyer * Alternatively, use advanced API to set specific properties. 7442b9c00cbSConrad Meyer * 7452b9c00cbSConrad Meyer * Use ZSTD_decompressStream() repetitively to consume your input. 7462b9c00cbSConrad Meyer * The function will update both `pos` fields. 7472b9c00cbSConrad Meyer * If `input.pos < input.size`, some input has not been consumed. 7482b9c00cbSConrad Meyer * It's up to the caller to present again remaining data. 7492b9c00cbSConrad Meyer * The function tries to flush all data decoded immediately, respecting output buffer size. 7502b9c00cbSConrad Meyer * If `output.pos < output.size`, decoder has flushed everything it could. 7512b9c00cbSConrad Meyer * But if `output.pos == output.size`, there might be some data left within internal buffers., 7522b9c00cbSConrad Meyer * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. 7532b9c00cbSConrad Meyer * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. 7542b9c00cbSConrad Meyer * @return : 0 when a frame is completely decoded and fully flushed, 7552b9c00cbSConrad Meyer * or an error code, which can be tested using ZSTD_isError(), 7562b9c00cbSConrad Meyer * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : 7572b9c00cbSConrad Meyer * the return value is a suggested next input size (just a hint for better latency) 7582b9c00cbSConrad Meyer * that will never request more than the remaining frame size. 7592b9c00cbSConrad Meyer * *******************************************************************************/ 7602b9c00cbSConrad Meyer 7612b9c00cbSConrad Meyer typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ 7622b9c00cbSConrad Meyer /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ 7632b9c00cbSConrad Meyer /*===== ZSTD_DStream management functions =====*/ 7642b9c00cbSConrad Meyer ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); 7652b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); 7662b9c00cbSConrad Meyer 7672b9c00cbSConrad Meyer /*===== Streaming decompression functions =====*/ 7682b9c00cbSConrad Meyer 7692b9c00cbSConrad Meyer /* This function is redundant with the advanced API and equivalent to: 7702b9c00cbSConrad Meyer * 771*37f1f268SConrad Meyer * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); 7722b9c00cbSConrad Meyer * ZSTD_DCtx_refDDict(zds, NULL); 7732b9c00cbSConrad Meyer */ 7742b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); 7752b9c00cbSConrad Meyer 7762b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); 7772b9c00cbSConrad Meyer 7782b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ 7792b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ 7802b9c00cbSConrad Meyer 7812b9c00cbSConrad Meyer 7822b9c00cbSConrad Meyer /************************** 7832b9c00cbSConrad Meyer * Simple dictionary API 7842b9c00cbSConrad Meyer ***************************/ 7852b9c00cbSConrad Meyer /*! ZSTD_compress_usingDict() : 7862b9c00cbSConrad Meyer * Compression at an explicit compression level using a Dictionary. 7872b9c00cbSConrad Meyer * A dictionary can be any arbitrary data segment (also called a prefix), 7882b9c00cbSConrad Meyer * or a buffer with specified information (see dictBuilder/zdict.h). 7892b9c00cbSConrad Meyer * Note : This function loads the dictionary, resulting in significant startup delay. 7902b9c00cbSConrad Meyer * It's intended for a dictionary used only once. 7912b9c00cbSConrad Meyer * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ 7922b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, 7932b9c00cbSConrad Meyer void* dst, size_t dstCapacity, 7942b9c00cbSConrad Meyer const void* src, size_t srcSize, 7952b9c00cbSConrad Meyer const void* dict,size_t dictSize, 7962b9c00cbSConrad Meyer int compressionLevel); 7972b9c00cbSConrad Meyer 7982b9c00cbSConrad Meyer /*! ZSTD_decompress_usingDict() : 7992b9c00cbSConrad Meyer * Decompression using a known Dictionary. 8002b9c00cbSConrad Meyer * Dictionary must be identical to the one used during compression. 8012b9c00cbSConrad Meyer * Note : This function loads the dictionary, resulting in significant startup delay. 8022b9c00cbSConrad Meyer * It's intended for a dictionary used only once. 8032b9c00cbSConrad Meyer * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ 8042b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, 8052b9c00cbSConrad Meyer void* dst, size_t dstCapacity, 8062b9c00cbSConrad Meyer const void* src, size_t srcSize, 8072b9c00cbSConrad Meyer const void* dict,size_t dictSize); 8082b9c00cbSConrad Meyer 8092b9c00cbSConrad Meyer 8102b9c00cbSConrad Meyer /*********************************** 8112b9c00cbSConrad Meyer * Bulk processing dictionary API 8122b9c00cbSConrad Meyer **********************************/ 8132b9c00cbSConrad Meyer typedef struct ZSTD_CDict_s ZSTD_CDict; 8142b9c00cbSConrad Meyer 8152b9c00cbSConrad Meyer /*! ZSTD_createCDict() : 8169cbefe25SConrad Meyer * When compressing multiple messages or blocks using the same dictionary, 8179cbefe25SConrad Meyer * it's recommended to digest the dictionary only once, since it's a costly operation. 8189cbefe25SConrad Meyer * ZSTD_createCDict() will create a state from digesting a dictionary. 8199cbefe25SConrad Meyer * The resulting state can be used for future compression operations with very limited startup cost. 8202b9c00cbSConrad Meyer * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. 8219cbefe25SConrad Meyer * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. 8229cbefe25SConrad Meyer * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. 8239cbefe25SConrad Meyer * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, 8249cbefe25SConrad Meyer * in which case the only thing that it transports is the @compressionLevel. 8259cbefe25SConrad Meyer * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, 8269cbefe25SConrad Meyer * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ 8272b9c00cbSConrad Meyer ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, 8282b9c00cbSConrad Meyer int compressionLevel); 8292b9c00cbSConrad Meyer 8302b9c00cbSConrad Meyer /*! ZSTD_freeCDict() : 8312b9c00cbSConrad Meyer * Function frees memory allocated by ZSTD_createCDict(). */ 8322b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); 8332b9c00cbSConrad Meyer 8342b9c00cbSConrad Meyer /*! ZSTD_compress_usingCDict() : 8352b9c00cbSConrad Meyer * Compression using a digested Dictionary. 8362b9c00cbSConrad Meyer * Recommended when same dictionary is used multiple times. 8372b9c00cbSConrad Meyer * Note : compression level is _decided at dictionary creation time_, 8382b9c00cbSConrad Meyer * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ 8392b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, 8402b9c00cbSConrad Meyer void* dst, size_t dstCapacity, 8412b9c00cbSConrad Meyer const void* src, size_t srcSize, 8422b9c00cbSConrad Meyer const ZSTD_CDict* cdict); 8432b9c00cbSConrad Meyer 8442b9c00cbSConrad Meyer 8452b9c00cbSConrad Meyer typedef struct ZSTD_DDict_s ZSTD_DDict; 8462b9c00cbSConrad Meyer 8472b9c00cbSConrad Meyer /*! ZSTD_createDDict() : 8482b9c00cbSConrad Meyer * Create a digested dictionary, ready to start decompression operation without startup delay. 8492b9c00cbSConrad Meyer * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ 8502b9c00cbSConrad Meyer ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); 8512b9c00cbSConrad Meyer 8522b9c00cbSConrad Meyer /*! ZSTD_freeDDict() : 8532b9c00cbSConrad Meyer * Function frees memory allocated with ZSTD_createDDict() */ 8542b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); 8552b9c00cbSConrad Meyer 8562b9c00cbSConrad Meyer /*! ZSTD_decompress_usingDDict() : 8572b9c00cbSConrad Meyer * Decompression using a digested Dictionary. 8582b9c00cbSConrad Meyer * Recommended when same dictionary is used multiple times. */ 8592b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, 8602b9c00cbSConrad Meyer void* dst, size_t dstCapacity, 8612b9c00cbSConrad Meyer const void* src, size_t srcSize, 8622b9c00cbSConrad Meyer const ZSTD_DDict* ddict); 8632b9c00cbSConrad Meyer 8642b9c00cbSConrad Meyer 8652b9c00cbSConrad Meyer /******************************** 8662b9c00cbSConrad Meyer * Dictionary helper functions 8672b9c00cbSConrad Meyer *******************************/ 8682b9c00cbSConrad Meyer 8692b9c00cbSConrad Meyer /*! ZSTD_getDictID_fromDict() : 8702b9c00cbSConrad Meyer * Provides the dictID stored within dictionary. 8712b9c00cbSConrad Meyer * if @return == 0, the dictionary is not conformant with Zstandard specification. 8722b9c00cbSConrad Meyer * It can still be loaded, but as a content-only dictionary. */ 8732b9c00cbSConrad Meyer ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); 8742b9c00cbSConrad Meyer 8752b9c00cbSConrad Meyer /*! ZSTD_getDictID_fromDDict() : 8762b9c00cbSConrad Meyer * Provides the dictID of the dictionary loaded into `ddict`. 8772b9c00cbSConrad Meyer * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. 8782b9c00cbSConrad Meyer * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ 8792b9c00cbSConrad Meyer ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); 8802b9c00cbSConrad Meyer 8812b9c00cbSConrad Meyer /*! ZSTD_getDictID_fromFrame() : 8822b9c00cbSConrad Meyer * Provides the dictID required to decompressed the frame stored within `src`. 8832b9c00cbSConrad Meyer * If @return == 0, the dictID could not be decoded. 8842b9c00cbSConrad Meyer * This could for one of the following reasons : 8852b9c00cbSConrad Meyer * - The frame does not require a dictionary to be decoded (most common case). 8862b9c00cbSConrad Meyer * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. 8872b9c00cbSConrad Meyer * Note : this use case also happens when using a non-conformant dictionary. 8882b9c00cbSConrad Meyer * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). 8892b9c00cbSConrad Meyer * - This is not a Zstandard frame. 8902b9c00cbSConrad Meyer * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ 8912b9c00cbSConrad Meyer ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); 8922b9c00cbSConrad Meyer 8932b9c00cbSConrad Meyer 8942b9c00cbSConrad Meyer /******************************************************************************* 8952b9c00cbSConrad Meyer * Advanced dictionary and prefix API 8962b9c00cbSConrad Meyer * 8972b9c00cbSConrad Meyer * This API allows dictionaries to be used with ZSTD_compress2(), 8982b9c00cbSConrad Meyer * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and 8992b9c00cbSConrad Meyer * only reset with the context is reset with ZSTD_reset_parameters or 9002b9c00cbSConrad Meyer * ZSTD_reset_session_and_parameters. Prefixes are single-use. 9012b9c00cbSConrad Meyer ******************************************************************************/ 9022b9c00cbSConrad Meyer 9032b9c00cbSConrad Meyer 904a0483764SConrad Meyer /*! ZSTD_CCtx_loadDictionary() : 905a0483764SConrad Meyer * Create an internal CDict from `dict` buffer. 906a0483764SConrad Meyer * Decompression will have to use same dictionary. 907a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 908a0483764SConrad Meyer * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, 909a0483764SConrad Meyer * meaning "return to no-dictionary mode". 910a0483764SConrad Meyer * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. 911a0483764SConrad Meyer * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). 912a0483764SConrad Meyer * Note 2 : Loading a dictionary involves building tables. 913a0483764SConrad Meyer * It's also a CPU consuming operation, with non-negligible impact on latency. 914a0483764SConrad Meyer * Tables are dependent on compression parameters, and for this reason, 915a0483764SConrad Meyer * compression parameters can no longer be changed after loading a dictionary. 916a0483764SConrad Meyer * Note 3 :`dict` content will be copied internally. 917a0483764SConrad Meyer * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. 918a0483764SConrad Meyer * In such a case, dictionary buffer must outlive its users. 919a0483764SConrad Meyer * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() 920a0483764SConrad Meyer * to precisely select how dictionary content must be interpreted. */ 921a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); 922a0483764SConrad Meyer 923a0483764SConrad Meyer /*! ZSTD_CCtx_refCDict() : 924a0483764SConrad Meyer * Reference a prepared dictionary, to be used for all next compressed frames. 925a0483764SConrad Meyer * Note that compression parameters are enforced from within CDict, 9262b9c00cbSConrad Meyer * and supersede any compression parameter previously set within CCtx. 9272b9c00cbSConrad Meyer * The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. 9282b9c00cbSConrad Meyer * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. 929a0483764SConrad Meyer * The dictionary will remain valid for future compressed frames using same CCtx. 930a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 931a0483764SConrad Meyer * Special : Referencing a NULL CDict means "return to no-dictionary mode". 932a0483764SConrad Meyer * Note 1 : Currently, only one dictionary can be managed. 933a0483764SConrad Meyer * Referencing a new dictionary effectively "discards" any previous one. 934a0483764SConrad Meyer * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ 935a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); 936a0483764SConrad Meyer 937a0483764SConrad Meyer /*! ZSTD_CCtx_refPrefix() : 938a0483764SConrad Meyer * Reference a prefix (single-usage dictionary) for next compressed frame. 939a0483764SConrad Meyer * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). 940a0483764SConrad Meyer * Decompression will need same prefix to properly regenerate data. 941a0483764SConrad Meyer * Compressing with a prefix is similar in outcome as performing a diff and compressing it, 942a0483764SConrad Meyer * but performs much faster, especially during decompression (compression speed is tunable with compression level). 943a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 944a0483764SConrad Meyer * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary 945a0483764SConrad Meyer * Note 1 : Prefix buffer is referenced. It **must** outlive compression. 946a0483764SConrad Meyer * Its content must remain unmodified during compression. 947a0483764SConrad Meyer * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, 948a0483764SConrad Meyer * ensure that the window size is large enough to contain the entire source. 949a0483764SConrad Meyer * See ZSTD_c_windowLog. 950a0483764SConrad Meyer * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. 951a0483764SConrad Meyer * It's a CPU consuming operation, with non-negligible impact on latency. 952a0483764SConrad Meyer * If there is a need to use the same prefix multiple times, consider loadDictionary instead. 9539cbefe25SConrad Meyer * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). 954a0483764SConrad Meyer * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ 955a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, 956a0483764SConrad Meyer const void* prefix, size_t prefixSize); 957a0483764SConrad Meyer 958a0483764SConrad Meyer /*! ZSTD_DCtx_loadDictionary() : 959a0483764SConrad Meyer * Create an internal DDict from dict buffer, 960a0483764SConrad Meyer * to be used to decompress next frames. 961a0483764SConrad Meyer * The dictionary remains valid for all future frames, until explicitly invalidated. 962a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 963a0483764SConrad Meyer * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, 964a0483764SConrad Meyer * meaning "return to no-dictionary mode". 965a0483764SConrad Meyer * Note 1 : Loading a dictionary involves building tables, 966a0483764SConrad Meyer * which has a non-negligible impact on CPU usage and latency. 967a0483764SConrad Meyer * It's recommended to "load once, use many times", to amortize the cost 968a0483764SConrad Meyer * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. 969a0483764SConrad Meyer * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. 970a0483764SConrad Meyer * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of 971a0483764SConrad Meyer * how dictionary content is loaded and interpreted. 972a0483764SConrad Meyer */ 973a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); 974a0483764SConrad Meyer 975a0483764SConrad Meyer /*! ZSTD_DCtx_refDDict() : 976a0483764SConrad Meyer * Reference a prepared dictionary, to be used to decompress next frames. 977a0483764SConrad Meyer * The dictionary remains active for decompression of future frames using same DCtx. 978a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 979a0483764SConrad Meyer * Note 1 : Currently, only one dictionary can be managed. 980a0483764SConrad Meyer * Referencing a new dictionary effectively "discards" any previous one. 981a0483764SConrad Meyer * Special: referencing a NULL DDict means "return to no-dictionary mode". 982a0483764SConrad Meyer * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. 983a0483764SConrad Meyer */ 984a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 985a0483764SConrad Meyer 986a0483764SConrad Meyer /*! ZSTD_DCtx_refPrefix() : 987a0483764SConrad Meyer * Reference a prefix (single-usage dictionary) to decompress next frame. 988a0483764SConrad Meyer * This is the reverse operation of ZSTD_CCtx_refPrefix(), 989a0483764SConrad Meyer * and must use the same prefix as the one used during compression. 990a0483764SConrad Meyer * Prefix is **only used once**. Reference is discarded at end of frame. 991a0483764SConrad Meyer * End of frame is reached when ZSTD_decompressStream() returns 0. 992a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 993a0483764SConrad Meyer * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary 994a0483764SConrad Meyer * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. 995a0483764SConrad Meyer * Prefix buffer must remain unmodified up to the end of frame, 996a0483764SConrad Meyer * reached when ZSTD_decompressStream() returns 0. 9979cbefe25SConrad Meyer * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). 998a0483764SConrad Meyer * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) 999a0483764SConrad Meyer * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. 1000a0483764SConrad Meyer * A full dictionary is more costly, as it requires building tables. 1001a0483764SConrad Meyer */ 1002a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, 1003a0483764SConrad Meyer const void* prefix, size_t prefixSize); 1004a0483764SConrad Meyer 10052b9c00cbSConrad Meyer /* === Memory management === */ 10062b9c00cbSConrad Meyer 10072b9c00cbSConrad Meyer /*! ZSTD_sizeof_*() : 10082b9c00cbSConrad Meyer * These functions give the _current_ memory usage of selected object. 10092b9c00cbSConrad Meyer * Note that object memory usage can evolve (increase or decrease) over time. */ 10102b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); 10112b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); 10122b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); 10132b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); 10142b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); 10152b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); 10162b9c00cbSConrad Meyer 10172b9c00cbSConrad Meyer #endif /* ZSTD_H_235446 */ 1018a0483764SConrad Meyer 1019a0483764SConrad Meyer 10204d3f1eafSConrad Meyer /* ************************************************************************************** 10212b9c00cbSConrad Meyer * ADVANCED AND EXPERIMENTAL FUNCTIONS 10222b9c00cbSConrad Meyer **************************************************************************************** 10232b9c00cbSConrad Meyer * The definitions in the following section are considered experimental. 10242b9c00cbSConrad Meyer * They are provided for advanced scenarios. 10252b9c00cbSConrad Meyer * They should never be used with a dynamic library, as prototypes may change in the future. 10262b9c00cbSConrad Meyer * Use them only in association with static linking. 10272b9c00cbSConrad Meyer * ***************************************************************************************/ 10282b9c00cbSConrad Meyer 10292b9c00cbSConrad Meyer #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) 10302b9c00cbSConrad Meyer #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY 1031a0483764SConrad Meyer 1032a0483764SConrad Meyer /**************************************************************************************** 1033a0483764SConrad Meyer * experimental API (static linking only) 1034a0483764SConrad Meyer **************************************************************************************** 1035a0483764SConrad Meyer * The following symbols and constants 1036a0483764SConrad Meyer * are not planned to join "stable API" status in the near future. 1037a0483764SConrad Meyer * They can still change in future versions. 1038a0483764SConrad Meyer * Some of them are planned to remain in the static_only section indefinitely. 1039a0483764SConrad Meyer * Some of them might be removed in the future (especially when redundant with existing stable functions) 1040a0483764SConrad Meyer * ***************************************************************************************/ 1041a0483764SConrad Meyer 10429cbefe25SConrad Meyer #define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ 10439cbefe25SConrad Meyer #define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) 1044a0483764SConrad Meyer #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ 1045a0483764SConrad Meyer #define ZSTD_SKIPPABLEHEADERSIZE 8 1046a0483764SConrad Meyer 1047a0483764SConrad Meyer /* compression parameter bounds */ 10480c16b537SWarner Losh #define ZSTD_WINDOWLOG_MAX_32 30 10490c16b537SWarner Losh #define ZSTD_WINDOWLOG_MAX_64 31 1050a0483764SConrad Meyer #define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) 10510c16b537SWarner Losh #define ZSTD_WINDOWLOG_MIN 10 105219fcbaf1SConrad Meyer #define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) 10530c16b537SWarner Losh #define ZSTD_HASHLOG_MIN 6 105419fcbaf1SConrad Meyer #define ZSTD_CHAINLOG_MAX_32 29 105519fcbaf1SConrad Meyer #define ZSTD_CHAINLOG_MAX_64 30 1056a0483764SConrad Meyer #define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) 10570c16b537SWarner Losh #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN 10580c16b537SWarner Losh #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) 10590c16b537SWarner Losh #define ZSTD_SEARCHLOG_MIN 1 1060a0483764SConrad Meyer #define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ 1061a0483764SConrad Meyer #define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ 10620f743729SConrad Meyer #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX 10630f743729SConrad Meyer #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ 1064a0483764SConrad Meyer #define ZSTD_STRATEGY_MIN ZSTD_fast 1065a0483764SConrad Meyer #define ZSTD_STRATEGY_MAX ZSTD_btultra2 1066a0483764SConrad Meyer 1067a0483764SConrad Meyer 1068a0483764SConrad Meyer #define ZSTD_OVERLAPLOG_MIN 0 1069a0483764SConrad Meyer #define ZSTD_OVERLAPLOG_MAX 9 1070a0483764SConrad Meyer 1071a0483764SConrad Meyer #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame 1072a0483764SConrad Meyer * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size, 1073a0483764SConrad Meyer * to preserve host's memory from unreasonable requirements. 10742b9c00cbSConrad Meyer * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,). 1075a0483764SConrad Meyer * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */ 1076a0483764SConrad Meyer 1077a0483764SConrad Meyer 1078a0483764SConrad Meyer /* LDM parameter bounds */ 1079a0483764SConrad Meyer #define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN 1080a0483764SConrad Meyer #define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX 10810f743729SConrad Meyer #define ZSTD_LDM_MINMATCH_MIN 4 1082a0483764SConrad Meyer #define ZSTD_LDM_MINMATCH_MAX 4096 1083a0483764SConrad Meyer #define ZSTD_LDM_BUCKETSIZELOG_MIN 1 10840c16b537SWarner Losh #define ZSTD_LDM_BUCKETSIZELOG_MAX 8 1085a0483764SConrad Meyer #define ZSTD_LDM_HASHRATELOG_MIN 0 1086a0483764SConrad Meyer #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) 10870c16b537SWarner Losh 10884d3f1eafSConrad Meyer /* Advanced parameter bounds */ 10894d3f1eafSConrad Meyer #define ZSTD_TARGETCBLOCKSIZE_MIN 64 10904d3f1eafSConrad Meyer #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX 10919cbefe25SConrad Meyer #define ZSTD_SRCSIZEHINT_MIN 0 10929cbefe25SConrad Meyer #define ZSTD_SRCSIZEHINT_MAX INT_MAX 10934d3f1eafSConrad Meyer 1094a0483764SConrad Meyer /* internal */ 1095a0483764SConrad Meyer #define ZSTD_HASHLOG3_MAX 17 10960c16b537SWarner Losh 10970f743729SConrad Meyer 10980c16b537SWarner Losh /* --- Advanced types --- */ 1099a0483764SConrad Meyer 1100a0483764SConrad Meyer typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; 11010c16b537SWarner Losh 11020c16b537SWarner Losh typedef struct { 11039cbefe25SConrad Meyer unsigned int matchPos; /* Match pos in dst */ 11049cbefe25SConrad Meyer /* If seqDef.offset > 3, then this is seqDef.offset - 3 11059cbefe25SConrad Meyer * If seqDef.offset < 3, then this is the corresponding repeat offset 11069cbefe25SConrad Meyer * But if seqDef.offset < 3 and litLength == 0, this is the 11079cbefe25SConrad Meyer * repeat offset before the corresponding repeat offset 11089cbefe25SConrad Meyer * And if seqDef.offset == 3 and litLength == 0, this is the 11099cbefe25SConrad Meyer * most recent repeat offset - 1 11109cbefe25SConrad Meyer */ 11119cbefe25SConrad Meyer unsigned int offset; 11129cbefe25SConrad Meyer unsigned int litLength; /* Literal length */ 11139cbefe25SConrad Meyer unsigned int matchLength; /* Match length */ 11149cbefe25SConrad Meyer /* 0 when seq not rep and seqDef.offset otherwise 11159cbefe25SConrad Meyer * when litLength == 0 this will be <= 4, otherwise <= 3 like normal 11169cbefe25SConrad Meyer */ 11179cbefe25SConrad Meyer unsigned int rep; 11189cbefe25SConrad Meyer } ZSTD_Sequence; 11199cbefe25SConrad Meyer 11209cbefe25SConrad Meyer typedef struct { 11210c16b537SWarner Losh unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ 11220c16b537SWarner Losh unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ 11230c16b537SWarner Losh unsigned hashLog; /**< dispatch table : larger == faster, more memory */ 11240c16b537SWarner Losh unsigned searchLog; /**< nb of searches : larger == more compression, slower */ 1125a0483764SConrad Meyer unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ 11260c16b537SWarner Losh unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ 1127a0483764SConrad Meyer ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ 11280c16b537SWarner Losh } ZSTD_compressionParameters; 11290c16b537SWarner Losh 11300c16b537SWarner Losh typedef struct { 1131a0483764SConrad Meyer int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ 1132a0483764SConrad Meyer int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ 1133a0483764SConrad Meyer int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ 11340c16b537SWarner Losh } ZSTD_frameParameters; 11350c16b537SWarner Losh 11360c16b537SWarner Losh typedef struct { 11370c16b537SWarner Losh ZSTD_compressionParameters cParams; 11380c16b537SWarner Losh ZSTD_frameParameters fParams; 11390c16b537SWarner Losh } ZSTD_parameters; 11400c16b537SWarner Losh 114119fcbaf1SConrad Meyer typedef enum { 114219fcbaf1SConrad Meyer ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ 1143a0483764SConrad Meyer ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ 1144a0483764SConrad Meyer ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ 114519fcbaf1SConrad Meyer } ZSTD_dictContentType_e; 114619fcbaf1SConrad Meyer 114719fcbaf1SConrad Meyer typedef enum { 114819fcbaf1SConrad Meyer ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ 11499cbefe25SConrad Meyer ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ 115019fcbaf1SConrad Meyer } ZSTD_dictLoadMethod_e; 115119fcbaf1SConrad Meyer 1152a0483764SConrad Meyer typedef enum { 1153a0483764SConrad Meyer ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ 11549cbefe25SConrad Meyer ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. 1155a0483764SConrad Meyer * Useful to save 4 bytes per generated frame. 1156a0483764SConrad Meyer * Decoder cannot recognise automatically this format, requiring this instruction. */ 1157a0483764SConrad Meyer } ZSTD_format_e; 1158a0483764SConrad Meyer 1159a0483764SConrad Meyer typedef enum { 1160a0483764SConrad Meyer /* Note: this enum and the behavior it controls are effectively internal 1161a0483764SConrad Meyer * implementation details of the compressor. They are expected to continue 1162a0483764SConrad Meyer * to evolve and should be considered only in the context of extremely 1163a0483764SConrad Meyer * advanced performance tuning. 1164a0483764SConrad Meyer * 11659cbefe25SConrad Meyer * Zstd currently supports the use of a CDict in three ways: 1166a0483764SConrad Meyer * 1167a0483764SConrad Meyer * - The contents of the CDict can be copied into the working context. This 1168a0483764SConrad Meyer * means that the compression can search both the dictionary and input 1169a0483764SConrad Meyer * while operating on a single set of internal tables. This makes 1170a0483764SConrad Meyer * the compression faster per-byte of input. However, the initial copy of 1171a0483764SConrad Meyer * the CDict's tables incurs a fixed cost at the beginning of the 1172a0483764SConrad Meyer * compression. For small compressions (< 8 KB), that copy can dominate 1173a0483764SConrad Meyer * the cost of the compression. 1174a0483764SConrad Meyer * 1175a0483764SConrad Meyer * - The CDict's tables can be used in-place. In this model, compression is 1176a0483764SConrad Meyer * slower per input byte, because the compressor has to search two sets of 1177a0483764SConrad Meyer * tables. However, this model incurs no start-up cost (as long as the 1178a0483764SConrad Meyer * working context's tables can be reused). For small inputs, this can be 1179a0483764SConrad Meyer * faster than copying the CDict's tables. 1180a0483764SConrad Meyer * 11819cbefe25SConrad Meyer * - The CDict's tables are not used at all, and instead we use the working 11829cbefe25SConrad Meyer * context alone to reload the dictionary and use params based on the source 11839cbefe25SConrad Meyer * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). 11849cbefe25SConrad Meyer * This method is effective when the dictionary sizes are very small relative 11859cbefe25SConrad Meyer * to the input size, and the input size is fairly large to begin with. 11869cbefe25SConrad Meyer * 1187a0483764SConrad Meyer * Zstd has a simple internal heuristic that selects which strategy to use 1188a0483764SConrad Meyer * at the beginning of a compression. However, if experimentation shows that 1189a0483764SConrad Meyer * Zstd is making poor choices, it is possible to override that choice with 1190a0483764SConrad Meyer * this enum. 1191a0483764SConrad Meyer */ 1192a0483764SConrad Meyer ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ 1193a0483764SConrad Meyer ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ 1194a0483764SConrad Meyer ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ 11959cbefe25SConrad Meyer ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ 1196a0483764SConrad Meyer } ZSTD_dictAttachPref_e; 11970c16b537SWarner Losh 11982b9c00cbSConrad Meyer typedef enum { 11992b9c00cbSConrad Meyer ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. 12002b9c00cbSConrad Meyer * Negative compression levels will be uncompressed, and positive compression 12012b9c00cbSConrad Meyer * levels will be compressed. */ 12022b9c00cbSConrad Meyer ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be 12032b9c00cbSConrad Meyer * emitted if Huffman compression is not profitable. */ 12049cbefe25SConrad Meyer ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ 12052b9c00cbSConrad Meyer } ZSTD_literalCompressionMode_e; 12062b9c00cbSConrad Meyer 12070c16b537SWarner Losh 12080c16b537SWarner Losh /*************************************** 12090c16b537SWarner Losh * Frame size functions 12100c16b537SWarner Losh ***************************************/ 12110c16b537SWarner Losh 12120c16b537SWarner Losh /*! ZSTD_findDecompressedSize() : 12132b9c00cbSConrad Meyer * `src` should point to the start of a series of ZSTD encoded and/or skippable frames 12140c16b537SWarner Losh * `srcSize` must be the _exact_ size of this series 12152b9c00cbSConrad Meyer * (i.e. there should be a frame boundary at `src + srcSize`) 12160c16b537SWarner Losh * @return : - decompressed size of all data in all successive frames 12170c16b537SWarner Losh * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN 12180c16b537SWarner Losh * - if an error occurred: ZSTD_CONTENTSIZE_ERROR 12190c16b537SWarner Losh * 12200c16b537SWarner Losh * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. 12210c16b537SWarner Losh * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. 12220c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 12230c16b537SWarner Losh * note 2 : decompressed size is always present when compression is done with ZSTD_compress() 12240c16b537SWarner Losh * note 3 : decompressed size can be very large (64-bits value), 12250c16b537SWarner Losh * potentially larger than what local system can handle as a single memory segment. 12260c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 12270c16b537SWarner Losh * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. 12280c16b537SWarner Losh * Always ensure result fits within application's authorized limits. 12290c16b537SWarner Losh * Each application can set its own limits. 12300c16b537SWarner Losh * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to 12310c16b537SWarner Losh * read each contained frame header. This is fast as most of the data is skipped, 12320c16b537SWarner Losh * however it does mean that all frame data must be present and valid. */ 12330c16b537SWarner Losh ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); 12340c16b537SWarner Losh 12354d3f1eafSConrad Meyer /*! ZSTD_decompressBound() : 12362b9c00cbSConrad Meyer * `src` should point to the start of a series of ZSTD encoded and/or skippable frames 12372b9c00cbSConrad Meyer * `srcSize` must be the _exact_ size of this series 12382b9c00cbSConrad Meyer * (i.e. there should be a frame boundary at `src + srcSize`) 12392b9c00cbSConrad Meyer * @return : - upper-bound for the decompressed size of all data in all successive frames 12402b9c00cbSConrad Meyer * - if an error occured: ZSTD_CONTENTSIZE_ERROR 12412b9c00cbSConrad Meyer * 12422b9c00cbSConrad Meyer * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. 12432b9c00cbSConrad Meyer * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. 12442b9c00cbSConrad Meyer * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. 12452b9c00cbSConrad Meyer * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: 12462b9c00cbSConrad Meyer * upper-bound = # blocks * min(128 KB, Window_Size) 12472b9c00cbSConrad Meyer */ 12482b9c00cbSConrad Meyer ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); 12492b9c00cbSConrad Meyer 12500c16b537SWarner Losh /*! ZSTD_frameHeaderSize() : 1251a0483764SConrad Meyer * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. 12520f743729SConrad Meyer * @return : size of the Frame Header, 12530f743729SConrad Meyer * or an error code (if srcSize is too small) */ 12540c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); 12550c16b537SWarner Losh 12569cbefe25SConrad Meyer /*! ZSTD_getSequences() : 12579cbefe25SConrad Meyer * Extract sequences from the sequence store 12589cbefe25SConrad Meyer * zc can be used to insert custom compression params. 12599cbefe25SConrad Meyer * This function invokes ZSTD_compress2 12609cbefe25SConrad Meyer * @return : number of sequences extracted 12619cbefe25SConrad Meyer */ 12629cbefe25SConrad Meyer ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, 12639cbefe25SConrad Meyer size_t outSeqsSize, const void* src, size_t srcSize); 12649cbefe25SConrad Meyer 12650c16b537SWarner Losh 12660c16b537SWarner Losh /*************************************** 126719fcbaf1SConrad Meyer * Memory management 12680c16b537SWarner Losh ***************************************/ 12690c16b537SWarner Losh 12700c16b537SWarner Losh /*! ZSTD_estimate*() : 1271*37f1f268SConrad Meyer * These functions make it possible to estimate memory usage 1272*37f1f268SConrad Meyer * of a future {D,C}Ctx, before its creation. 12739cbefe25SConrad Meyer * 1274*37f1f268SConrad Meyer * ZSTD_estimateCCtxSize() will provide a memory budget large enough 1275*37f1f268SConrad Meyer * for any compression level up to selected one. 1276*37f1f268SConrad Meyer * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate 1277*37f1f268SConrad Meyer * does not include space for a window buffer. 1278*37f1f268SConrad Meyer * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. 1279*37f1f268SConrad Meyer * The estimate will assume the input may be arbitrarily large, 1280*37f1f268SConrad Meyer * which is the worst case. 12819cbefe25SConrad Meyer * 1282*37f1f268SConrad Meyer * When srcSize can be bound by a known and rather "small" value, 1283*37f1f268SConrad Meyer * this fact can be used to provide a tighter estimation 1284*37f1f268SConrad Meyer * because the CCtx compression context will need less memory. 1285*37f1f268SConrad Meyer * This tighter estimation can be provided by more advanced functions 1286*37f1f268SConrad Meyer * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), 1287*37f1f268SConrad Meyer * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). 1288*37f1f268SConrad Meyer * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. 1289*37f1f268SConrad Meyer * 1290*37f1f268SConrad Meyer * Note 2 : only single-threaded compression is supported. 1291*37f1f268SConrad Meyer * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. 1292*37f1f268SConrad Meyer */ 12930c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); 12940c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); 12950c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); 12960c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); 12970c16b537SWarner Losh 12980c16b537SWarner Losh /*! ZSTD_estimateCStreamSize() : 12990c16b537SWarner Losh * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. 13000c16b537SWarner Losh * It will also consider src size to be arbitrarily "large", which is worst case. 13010c16b537SWarner Losh * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. 13020c16b537SWarner Losh * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. 13032b9c00cbSConrad Meyer * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. 130419fcbaf1SConrad Meyer * Note : CStream size estimation is only correct for single-threaded compression. 13050c16b537SWarner Losh * ZSTD_DStream memory budget depends on window Size. 13060c16b537SWarner Losh * This information can be passed manually, using ZSTD_estimateDStreamSize, 13070c16b537SWarner Losh * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); 13080c16b537SWarner Losh * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), 13090c16b537SWarner Losh * an internal ?Dict will be created, which additional size is not estimated here. 13100c16b537SWarner Losh * In this case, get total size by adding ZSTD_estimate?DictSize */ 13110c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); 13120c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); 13130c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); 13140c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); 13150c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); 13160c16b537SWarner Losh 13170c16b537SWarner Losh /*! ZSTD_estimate?DictSize() : 13180c16b537SWarner Losh * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). 131919fcbaf1SConrad Meyer * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). 132019fcbaf1SConrad Meyer * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. 13210c16b537SWarner Losh */ 13220c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); 13230c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); 13240c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); 13250c16b537SWarner Losh 132619fcbaf1SConrad Meyer /*! ZSTD_initStatic*() : 132719fcbaf1SConrad Meyer * Initialize an object using a pre-allocated fixed-size buffer. 132819fcbaf1SConrad Meyer * workspace: The memory area to emplace the object into. 132919fcbaf1SConrad Meyer * Provided pointer *must be 8-bytes aligned*. 133019fcbaf1SConrad Meyer * Buffer must outlive object. 133119fcbaf1SConrad Meyer * workspaceSize: Use ZSTD_estimate*Size() to determine 133219fcbaf1SConrad Meyer * how large workspace must be to support target scenario. 133319fcbaf1SConrad Meyer * @return : pointer to object (same address as workspace, just different type), 133419fcbaf1SConrad Meyer * or NULL if error (size too small, incorrect alignment, etc.) 133519fcbaf1SConrad Meyer * Note : zstd will never resize nor malloc() when using a static buffer. 133619fcbaf1SConrad Meyer * If the object requires more memory than available, 133719fcbaf1SConrad Meyer * zstd will just error out (typically ZSTD_error_memory_allocation). 133819fcbaf1SConrad Meyer * Note 2 : there is no corresponding "free" function. 133919fcbaf1SConrad Meyer * Since workspace is allocated externally, it must be freed externally too. 134019fcbaf1SConrad Meyer * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level 134119fcbaf1SConrad Meyer * into its associated cParams. 134219fcbaf1SConrad Meyer * Limitation 1 : currently not compatible with internal dictionary creation, triggered by 134319fcbaf1SConrad Meyer * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). 134419fcbaf1SConrad Meyer * Limitation 2 : static cctx currently not compatible with multi-threading. 134519fcbaf1SConrad Meyer * Limitation 3 : static dctx is incompatible with legacy support. 134619fcbaf1SConrad Meyer */ 134719fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); 134819fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ 134919fcbaf1SConrad Meyer 135019fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); 135119fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ 135219fcbaf1SConrad Meyer 135319fcbaf1SConrad Meyer ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( 135419fcbaf1SConrad Meyer void* workspace, size_t workspaceSize, 135519fcbaf1SConrad Meyer const void* dict, size_t dictSize, 135619fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 135719fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType, 135819fcbaf1SConrad Meyer ZSTD_compressionParameters cParams); 135919fcbaf1SConrad Meyer 136019fcbaf1SConrad Meyer ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( 136119fcbaf1SConrad Meyer void* workspace, size_t workspaceSize, 136219fcbaf1SConrad Meyer const void* dict, size_t dictSize, 136319fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 136419fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType); 136519fcbaf1SConrad Meyer 1366a0483764SConrad Meyer 136719fcbaf1SConrad Meyer /*! Custom memory allocation : 136819fcbaf1SConrad Meyer * These prototypes make it possible to pass your own allocation/free functions. 136919fcbaf1SConrad Meyer * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. 137019fcbaf1SConrad Meyer * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. 137119fcbaf1SConrad Meyer */ 137219fcbaf1SConrad Meyer typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); 137319fcbaf1SConrad Meyer typedef void (*ZSTD_freeFunction) (void* opaque, void* address); 137419fcbaf1SConrad Meyer typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; 137519fcbaf1SConrad Meyer static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ 137619fcbaf1SConrad Meyer 137719fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); 137819fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); 137919fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); 138019fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); 138119fcbaf1SConrad Meyer 138219fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, 138319fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 138419fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType, 138519fcbaf1SConrad Meyer ZSTD_compressionParameters cParams, 138619fcbaf1SConrad Meyer ZSTD_customMem customMem); 138719fcbaf1SConrad Meyer 138819fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, 138919fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 139019fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType, 139119fcbaf1SConrad Meyer ZSTD_customMem customMem); 139219fcbaf1SConrad Meyer 139319fcbaf1SConrad Meyer 13940c16b537SWarner Losh 13950c16b537SWarner Losh /*************************************** 13960c16b537SWarner Losh * Advanced compression functions 13970c16b537SWarner Losh ***************************************/ 13980c16b537SWarner Losh 13990c16b537SWarner Losh /*! ZSTD_createCDict_byReference() : 14000c16b537SWarner Losh * Create a digested dictionary for compression 1401a0483764SConrad Meyer * Dictionary content is just referenced, not duplicated. 1402a0483764SConrad Meyer * As a consequence, `dictBuffer` **must** outlive CDict, 14039cbefe25SConrad Meyer * and its content must remain unmodified throughout the lifetime of CDict. 14049cbefe25SConrad Meyer * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ 14050c16b537SWarner Losh ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); 14060c16b537SWarner Losh 14070c16b537SWarner Losh /*! ZSTD_getCParams() : 14080c16b537SWarner Losh * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. 14090c16b537SWarner Losh * `estimatedSrcSize` value is optional, select 0 if not known */ 14100c16b537SWarner Losh ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); 14110c16b537SWarner Losh 14120c16b537SWarner Losh /*! ZSTD_getParams() : 14130c16b537SWarner Losh * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. 1414052d3c12SConrad Meyer * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ 14150c16b537SWarner Losh ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); 14160c16b537SWarner Losh 14170c16b537SWarner Losh /*! ZSTD_checkCParams() : 14182b9c00cbSConrad Meyer * Ensure param values remain within authorized range. 14192b9c00cbSConrad Meyer * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ 14200c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); 14210c16b537SWarner Losh 14220c16b537SWarner Losh /*! ZSTD_adjustCParams() : 14230c16b537SWarner Losh * optimize params for a given `srcSize` and `dictSize`. 14242b9c00cbSConrad Meyer * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. 14252b9c00cbSConrad Meyer * `dictSize` must be `0` when there is no dictionary. 14262b9c00cbSConrad Meyer * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. 14272b9c00cbSConrad Meyer * This function never fails (wide contract) */ 14280c16b537SWarner Losh ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); 14290c16b537SWarner Losh 14300c16b537SWarner Losh /*! ZSTD_compress_advanced() : 14319cbefe25SConrad Meyer * Note : this function is now DEPRECATED. 14329cbefe25SConrad Meyer * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. 14339cbefe25SConrad Meyer * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ 14340c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, 14350c16b537SWarner Losh void* dst, size_t dstCapacity, 14360c16b537SWarner Losh const void* src, size_t srcSize, 14370c16b537SWarner Losh const void* dict,size_t dictSize, 14380c16b537SWarner Losh ZSTD_parameters params); 14390c16b537SWarner Losh 14400c16b537SWarner Losh /*! ZSTD_compress_usingCDict_advanced() : 14419cbefe25SConrad Meyer * Note : this function is now REDUNDANT. 14429cbefe25SConrad Meyer * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. 14439cbefe25SConrad Meyer * This prototype will be marked as deprecated and generate compilation warning in some future version */ 14440c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, 14450c16b537SWarner Losh void* dst, size_t dstCapacity, 14460c16b537SWarner Losh const void* src, size_t srcSize, 1447a0483764SConrad Meyer const ZSTD_CDict* cdict, 1448a0483764SConrad Meyer ZSTD_frameParameters fParams); 14490c16b537SWarner Losh 14500c16b537SWarner Losh 1451a0483764SConrad Meyer /*! ZSTD_CCtx_loadDictionary_byReference() : 1452a0483764SConrad Meyer * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. 1453a0483764SConrad Meyer * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ 1454a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); 1455a0483764SConrad Meyer 1456a0483764SConrad Meyer /*! ZSTD_CCtx_loadDictionary_advanced() : 1457a0483764SConrad Meyer * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over 1458a0483764SConrad Meyer * how to load the dictionary (by copy ? by reference ?) 1459a0483764SConrad Meyer * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ 1460a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); 1461a0483764SConrad Meyer 1462a0483764SConrad Meyer /*! ZSTD_CCtx_refPrefix_advanced() : 1463a0483764SConrad Meyer * Same as ZSTD_CCtx_refPrefix(), but gives finer control over 1464a0483764SConrad Meyer * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ 1465a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); 1466a0483764SConrad Meyer 1467a0483764SConrad Meyer /* === experimental parameters === */ 1468a0483764SConrad Meyer /* these parameters can be used with ZSTD_setParameter() 1469a0483764SConrad Meyer * they are not guaranteed to remain supported in the future */ 1470a0483764SConrad Meyer 1471a0483764SConrad Meyer /* Enables rsyncable mode, 1472a0483764SConrad Meyer * which makes compressed files more rsync friendly 1473a0483764SConrad Meyer * by adding periodic synchronization points to the compressed data. 1474a0483764SConrad Meyer * The target average block size is ZSTD_c_jobSize / 2. 1475a0483764SConrad Meyer * It's possible to modify the job size to increase or decrease 1476a0483764SConrad Meyer * the granularity of the synchronization point. 1477a0483764SConrad Meyer * Once the jobSize is smaller than the window size, 1478a0483764SConrad Meyer * it will result in compression ratio degradation. 1479a0483764SConrad Meyer * NOTE 1: rsyncable mode only works when multithreading is enabled. 1480a0483764SConrad Meyer * NOTE 2: rsyncable performs poorly in combination with long range mode, 1481a0483764SConrad Meyer * since it will decrease the effectiveness of synchronization points, 1482a0483764SConrad Meyer * though mileage may vary. 1483a0483764SConrad Meyer * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. 1484a0483764SConrad Meyer * If the selected compression level is already running significantly slower, 1485a0483764SConrad Meyer * the overall speed won't be significantly impacted. 1486a0483764SConrad Meyer */ 1487a0483764SConrad Meyer #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 1488a0483764SConrad Meyer 1489a0483764SConrad Meyer /* Select a compression format. 1490a0483764SConrad Meyer * The value must be of type ZSTD_format_e. 1491a0483764SConrad Meyer * See ZSTD_format_e enum definition for details */ 1492a0483764SConrad Meyer #define ZSTD_c_format ZSTD_c_experimentalParam2 1493a0483764SConrad Meyer 1494a0483764SConrad Meyer /* Force back-reference distances to remain < windowSize, 1495a0483764SConrad Meyer * even when referencing into Dictionary content (default:0) */ 1496a0483764SConrad Meyer #define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 1497a0483764SConrad Meyer 1498a0483764SConrad Meyer /* Controls whether the contents of a CDict 1499a0483764SConrad Meyer * are used in place, or copied into the working context. 1500a0483764SConrad Meyer * Accepts values from the ZSTD_dictAttachPref_e enum. 1501a0483764SConrad Meyer * See the comments on that enum for an explanation of the feature. */ 1502a0483764SConrad Meyer #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 1503a0483764SConrad Meyer 15042b9c00cbSConrad Meyer /* Controls how the literals are compressed (default is auto). 15052b9c00cbSConrad Meyer * The value must be of type ZSTD_literalCompressionMode_e. 15062b9c00cbSConrad Meyer * See ZSTD_literalCompressionMode_t enum definition for details. 15072b9c00cbSConrad Meyer */ 15082b9c00cbSConrad Meyer #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 15092b9c00cbSConrad Meyer 15104d3f1eafSConrad Meyer /* Tries to fit compressed block size to be around targetCBlockSize. 15114d3f1eafSConrad Meyer * No target when targetCBlockSize == 0. 15124d3f1eafSConrad Meyer * There is no guarantee on compressed block size (default:0) */ 15134d3f1eafSConrad Meyer #define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 15144d3f1eafSConrad Meyer 15159cbefe25SConrad Meyer /* User's best guess of source size. 15169cbefe25SConrad Meyer * Hint is not valid when srcSizeHint == 0. 15179cbefe25SConrad Meyer * There is no guarantee that hint is close to actual source size, 15189cbefe25SConrad Meyer * but compression ratio may regress significantly if guess considerably underestimates */ 15199cbefe25SConrad Meyer #define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 15209cbefe25SConrad Meyer 1521a0483764SConrad Meyer /*! ZSTD_CCtx_getParameter() : 1522a0483764SConrad Meyer * Get the requested compression parameter value, selected by enum ZSTD_cParameter, 1523a0483764SConrad Meyer * and store it into int* value. 1524a0483764SConrad Meyer * @return : 0, or an error code (which can be tested with ZSTD_isError()). 1525a0483764SConrad Meyer */ 1526a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); 1527a0483764SConrad Meyer 1528a0483764SConrad Meyer 1529a0483764SConrad Meyer /*! ZSTD_CCtx_params : 1530a0483764SConrad Meyer * Quick howto : 1531a0483764SConrad Meyer * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure 15322b9c00cbSConrad Meyer * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into 1533a0483764SConrad Meyer * an existing ZSTD_CCtx_params structure. 1534a0483764SConrad Meyer * This is similar to 1535a0483764SConrad Meyer * ZSTD_CCtx_setParameter(). 1536a0483764SConrad Meyer * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to 1537a0483764SConrad Meyer * an existing CCtx. 1538a0483764SConrad Meyer * These parameters will be applied to 1539a0483764SConrad Meyer * all subsequent frames. 1540a0483764SConrad Meyer * - ZSTD_compressStream2() : Do compression using the CCtx. 1541a0483764SConrad Meyer * - ZSTD_freeCCtxParams() : Free the memory. 1542a0483764SConrad Meyer * 1543a0483764SConrad Meyer * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() 1544a0483764SConrad Meyer * for static allocation of CCtx for single-threaded compression. 1545a0483764SConrad Meyer */ 1546a0483764SConrad Meyer ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); 1547a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); 1548a0483764SConrad Meyer 1549a0483764SConrad Meyer /*! ZSTD_CCtxParams_reset() : 1550a0483764SConrad Meyer * Reset params to default values. 1551a0483764SConrad Meyer */ 1552a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); 1553a0483764SConrad Meyer 1554a0483764SConrad Meyer /*! ZSTD_CCtxParams_init() : 1555a0483764SConrad Meyer * Initializes the compression parameters of cctxParams according to 1556a0483764SConrad Meyer * compression level. All other parameters are reset to their default values. 1557a0483764SConrad Meyer */ 1558a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); 1559a0483764SConrad Meyer 1560a0483764SConrad Meyer /*! ZSTD_CCtxParams_init_advanced() : 1561a0483764SConrad Meyer * Initializes the compression and frame parameters of cctxParams according to 1562a0483764SConrad Meyer * params. All other parameters are reset to their default values. 1563a0483764SConrad Meyer */ 1564a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); 1565a0483764SConrad Meyer 15662b9c00cbSConrad Meyer /*! ZSTD_CCtxParams_setParameter() : 1567a0483764SConrad Meyer * Similar to ZSTD_CCtx_setParameter. 1568a0483764SConrad Meyer * Set one compression parameter, selected by enum ZSTD_cParameter. 1569a0483764SConrad Meyer * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). 1570a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 1571a0483764SConrad Meyer */ 15722b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); 1573a0483764SConrad Meyer 15742b9c00cbSConrad Meyer /*! ZSTD_CCtxParams_getParameter() : 1575a0483764SConrad Meyer * Similar to ZSTD_CCtx_getParameter. 1576a0483764SConrad Meyer * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. 1577a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 1578a0483764SConrad Meyer */ 15792b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); 1580a0483764SConrad Meyer 1581a0483764SConrad Meyer /*! ZSTD_CCtx_setParametersUsingCCtxParams() : 1582a0483764SConrad Meyer * Apply a set of ZSTD_CCtx_params to the compression context. 1583a0483764SConrad Meyer * This can be done even after compression is started, 1584a0483764SConrad Meyer * if nbWorkers==0, this will have no impact until a new compression is started. 1585a0483764SConrad Meyer * if nbWorkers>=1, new parameters will be picked up at next job, 1586a0483764SConrad Meyer * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). 1587a0483764SConrad Meyer */ 1588a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( 1589a0483764SConrad Meyer ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); 1590a0483764SConrad Meyer 1591a0483764SConrad Meyer /*! ZSTD_compressStream2_simpleArgs() : 1592a0483764SConrad Meyer * Same as ZSTD_compressStream2(), 1593a0483764SConrad Meyer * but using only integral types as arguments. 1594a0483764SConrad Meyer * This variant might be helpful for binders from dynamic languages 1595a0483764SConrad Meyer * which have troubles handling structures containing memory pointers. 1596a0483764SConrad Meyer */ 1597a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( 1598a0483764SConrad Meyer ZSTD_CCtx* cctx, 1599a0483764SConrad Meyer void* dst, size_t dstCapacity, size_t* dstPos, 1600a0483764SConrad Meyer const void* src, size_t srcSize, size_t* srcPos, 1601a0483764SConrad Meyer ZSTD_EndDirective endOp); 1602a0483764SConrad Meyer 1603a0483764SConrad Meyer 1604a0483764SConrad Meyer /*************************************** 1605a0483764SConrad Meyer * Advanced decompression functions 1606a0483764SConrad Meyer ***************************************/ 16070c16b537SWarner Losh 16080c16b537SWarner Losh /*! ZSTD_isFrame() : 16090c16b537SWarner Losh * Tells if the content of `buffer` starts with a valid Frame Identifier. 16100c16b537SWarner Losh * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. 16110c16b537SWarner Losh * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. 16120c16b537SWarner Losh * Note 3 : Skippable Frame Identifiers are considered valid. */ 16130c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); 16140c16b537SWarner Losh 16150c16b537SWarner Losh /*! ZSTD_createDDict_byReference() : 16160c16b537SWarner Losh * Create a digested dictionary, ready to start decompression operation without startup delay. 16170c16b537SWarner Losh * Dictionary content is referenced, and therefore stays in dictBuffer. 16180c16b537SWarner Losh * It is important that dictBuffer outlives DDict, 16190c16b537SWarner Losh * it must remain read accessible throughout the lifetime of DDict */ 16200c16b537SWarner Losh ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); 16210c16b537SWarner Losh 1622a0483764SConrad Meyer /*! ZSTD_DCtx_loadDictionary_byReference() : 1623a0483764SConrad Meyer * Same as ZSTD_DCtx_loadDictionary(), 1624a0483764SConrad Meyer * but references `dict` content instead of copying it into `dctx`. 1625a0483764SConrad Meyer * This saves memory if `dict` remains around., 1626a0483764SConrad Meyer * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ 1627a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); 1628a0483764SConrad Meyer 1629a0483764SConrad Meyer /*! ZSTD_DCtx_loadDictionary_advanced() : 1630a0483764SConrad Meyer * Same as ZSTD_DCtx_loadDictionary(), 1631a0483764SConrad Meyer * but gives direct control over 1632a0483764SConrad Meyer * how to load the dictionary (by copy ? by reference ?) 1633a0483764SConrad Meyer * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ 1634a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); 1635a0483764SConrad Meyer 1636a0483764SConrad Meyer /*! ZSTD_DCtx_refPrefix_advanced() : 1637a0483764SConrad Meyer * Same as ZSTD_DCtx_refPrefix(), but gives finer control over 1638a0483764SConrad Meyer * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ 1639a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); 1640a0483764SConrad Meyer 1641a0483764SConrad Meyer /*! ZSTD_DCtx_setMaxWindowSize() : 1642a0483764SConrad Meyer * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. 1643a0483764SConrad Meyer * This protects a decoder context from reserving too much memory for itself (potential attack scenario). 1644a0483764SConrad Meyer * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. 1645a0483764SConrad Meyer * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) 1646a0483764SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()). 1647a0483764SConrad Meyer */ 1648a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); 1649a0483764SConrad Meyer 1650a0483764SConrad Meyer /* ZSTD_d_format 1651a0483764SConrad Meyer * experimental parameter, 1652a0483764SConrad Meyer * allowing selection between ZSTD_format_e input compression formats 1653a0483764SConrad Meyer */ 1654a0483764SConrad Meyer #define ZSTD_d_format ZSTD_d_experimentalParam1 1655*37f1f268SConrad Meyer /* ZSTD_d_stableOutBuffer 1656*37f1f268SConrad Meyer * Experimental parameter. 1657*37f1f268SConrad Meyer * Default is 0 == disabled. Set to 1 to enable. 1658*37f1f268SConrad Meyer * 1659*37f1f268SConrad Meyer * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same 1660*37f1f268SConrad Meyer * between calls, except for the modifications that zstd makes to pos (the 1661*37f1f268SConrad Meyer * caller must not modify pos). This is checked by the decompressor, and 1662*37f1f268SConrad Meyer * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer 1663*37f1f268SConrad Meyer * MUST be large enough to fit the entire decompressed frame. This will be 1664*37f1f268SConrad Meyer * checked when the frame content size is known. The data in the ZSTD_outBuffer 1665*37f1f268SConrad Meyer * in the range [dst, dst + pos) MUST not be modified during decompression 1666*37f1f268SConrad Meyer * or you will get data corruption. 1667*37f1f268SConrad Meyer * 1668*37f1f268SConrad Meyer * When this flags is enabled zstd won't allocate an output buffer, because 1669*37f1f268SConrad Meyer * it can write directly to the ZSTD_outBuffer, but it will still allocate 1670*37f1f268SConrad Meyer * an input buffer large enough to fit any compressed block. This will also 1671*37f1f268SConrad Meyer * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. 1672*37f1f268SConrad Meyer * If you need to avoid the input buffer allocation use the buffer-less 1673*37f1f268SConrad Meyer * streaming API. 1674*37f1f268SConrad Meyer * 1675*37f1f268SConrad Meyer * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using 1676*37f1f268SConrad Meyer * this flag is ALWAYS memory safe, and will never access out-of-bounds 1677*37f1f268SConrad Meyer * memory. However, decompression WILL fail if you violate the preconditions. 1678*37f1f268SConrad Meyer * 1679*37f1f268SConrad Meyer * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST 1680*37f1f268SConrad Meyer * not be modified during decompression or you will get data corruption. This 1681*37f1f268SConrad Meyer * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate 1682*37f1f268SConrad Meyer * matches. Normally zstd maintains its own buffer for this purpose, but passing 1683*37f1f268SConrad Meyer * this flag tells zstd to use the user provided buffer. 1684*37f1f268SConrad Meyer */ 1685*37f1f268SConrad Meyer #define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 1686a0483764SConrad Meyer 1687a0483764SConrad Meyer /*! ZSTD_DCtx_setFormat() : 1688a0483764SConrad Meyer * Instruct the decoder context about what kind of data to decode next. 1689a0483764SConrad Meyer * This instruction is mandatory to decode data without a fully-formed header, 1690a0483764SConrad Meyer * such ZSTD_f_zstd1_magicless for example. 1691a0483764SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ 1692a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); 1693a0483764SConrad Meyer 1694a0483764SConrad Meyer /*! ZSTD_decompressStream_simpleArgs() : 1695a0483764SConrad Meyer * Same as ZSTD_decompressStream(), 1696a0483764SConrad Meyer * but using only integral types as arguments. 1697a0483764SConrad Meyer * This can be helpful for binders from dynamic languages 1698a0483764SConrad Meyer * which have troubles handling structures containing memory pointers. 1699a0483764SConrad Meyer */ 1700a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( 1701a0483764SConrad Meyer ZSTD_DCtx* dctx, 1702a0483764SConrad Meyer void* dst, size_t dstCapacity, size_t* dstPos, 1703a0483764SConrad Meyer const void* src, size_t srcSize, size_t* srcPos); 1704a0483764SConrad Meyer 17050c16b537SWarner Losh 17060c16b537SWarner Losh /******************************************************************** 17070c16b537SWarner Losh * Advanced streaming functions 1708a0483764SConrad Meyer * Warning : most of these functions are now redundant with the Advanced API. 1709a0483764SConrad Meyer * Once Advanced API reaches "stable" status, 1710a0483764SConrad Meyer * redundant functions will be deprecated, and then at some point removed. 17110c16b537SWarner Losh ********************************************************************/ 17120c16b537SWarner Losh 17130c16b537SWarner Losh /*===== Advanced Streaming compression functions =====*/ 17142b9c00cbSConrad Meyer /**! ZSTD_initCStream_srcSize() : 17152b9c00cbSConrad Meyer * This function is deprecated, and equivalent to: 17162b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 17172b9c00cbSConrad Meyer * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) 17182b9c00cbSConrad Meyer * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); 17192b9c00cbSConrad Meyer * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); 17202b9c00cbSConrad Meyer * 17212b9c00cbSConrad Meyer * pledgedSrcSize must be correct. If it is not known at init time, use 17222b9c00cbSConrad Meyer * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, 17232b9c00cbSConrad Meyer * "0" also disables frame content size field. It may be enabled in the future. 17249cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 17252b9c00cbSConrad Meyer */ 17269cbefe25SConrad Meyer ZSTDLIB_API size_t 17279cbefe25SConrad Meyer ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, 17289cbefe25SConrad Meyer int compressionLevel, 17299cbefe25SConrad Meyer unsigned long long pledgedSrcSize); 17309cbefe25SConrad Meyer 17312b9c00cbSConrad Meyer /**! ZSTD_initCStream_usingDict() : 17322b9c00cbSConrad Meyer * This function is deprecated, and is equivalent to: 17332b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 17342b9c00cbSConrad Meyer * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); 17352b9c00cbSConrad Meyer * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); 17362b9c00cbSConrad Meyer * 17372b9c00cbSConrad Meyer * Creates of an internal CDict (incompatible with static CCtx), except if 17382b9c00cbSConrad Meyer * dict == NULL or dictSize < 8, in which case no dict is used. 17399cbefe25SConrad Meyer * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if 17402b9c00cbSConrad Meyer * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. 17419cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 17422b9c00cbSConrad Meyer */ 17439cbefe25SConrad Meyer ZSTDLIB_API size_t 17449cbefe25SConrad Meyer ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, 17459cbefe25SConrad Meyer const void* dict, size_t dictSize, 17469cbefe25SConrad Meyer int compressionLevel); 17479cbefe25SConrad Meyer 17482b9c00cbSConrad Meyer /**! ZSTD_initCStream_advanced() : 17492b9c00cbSConrad Meyer * This function is deprecated, and is approximately equivalent to: 17502b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 17519cbefe25SConrad Meyer * // Pseudocode: Set each zstd parameter and leave the rest as-is. 17529cbefe25SConrad Meyer * for ((param, value) : params) { 17539cbefe25SConrad Meyer * ZSTD_CCtx_setParameter(zcs, param, value); 17549cbefe25SConrad Meyer * } 17552b9c00cbSConrad Meyer * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); 17562b9c00cbSConrad Meyer * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); 17572b9c00cbSConrad Meyer * 17589cbefe25SConrad Meyer * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. 17599cbefe25SConrad Meyer * pledgedSrcSize must be correct. 17609cbefe25SConrad Meyer * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. 17619cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 17622b9c00cbSConrad Meyer */ 17639cbefe25SConrad Meyer ZSTDLIB_API size_t 17649cbefe25SConrad Meyer ZSTD_initCStream_advanced(ZSTD_CStream* zcs, 17659cbefe25SConrad Meyer const void* dict, size_t dictSize, 17669cbefe25SConrad Meyer ZSTD_parameters params, 17679cbefe25SConrad Meyer unsigned long long pledgedSrcSize); 17689cbefe25SConrad Meyer 17692b9c00cbSConrad Meyer /**! ZSTD_initCStream_usingCDict() : 17702b9c00cbSConrad Meyer * This function is deprecated, and equivalent to: 17712b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 17722b9c00cbSConrad Meyer * ZSTD_CCtx_refCDict(zcs, cdict); 17732b9c00cbSConrad Meyer * 17742b9c00cbSConrad Meyer * note : cdict will just be referenced, and must outlive compression session 17759cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 17762b9c00cbSConrad Meyer */ 17772b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); 17789cbefe25SConrad Meyer 17792b9c00cbSConrad Meyer /**! ZSTD_initCStream_usingCDict_advanced() : 17809cbefe25SConrad Meyer * This function is DEPRECATED, and is approximately equivalent to: 17812b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 17829cbefe25SConrad Meyer * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. 17839cbefe25SConrad Meyer * for ((fParam, value) : fParams) { 17849cbefe25SConrad Meyer * ZSTD_CCtx_setParameter(zcs, fParam, value); 17859cbefe25SConrad Meyer * } 17862b9c00cbSConrad Meyer * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); 17872b9c00cbSConrad Meyer * ZSTD_CCtx_refCDict(zcs, cdict); 17882b9c00cbSConrad Meyer * 17892b9c00cbSConrad Meyer * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. 17902b9c00cbSConrad Meyer * pledgedSrcSize must be correct. If srcSize is not known at init time, use 17912b9c00cbSConrad Meyer * value ZSTD_CONTENTSIZE_UNKNOWN. 17929cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 17932b9c00cbSConrad Meyer */ 17949cbefe25SConrad Meyer ZSTDLIB_API size_t 17959cbefe25SConrad Meyer ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, 17969cbefe25SConrad Meyer const ZSTD_CDict* cdict, 17979cbefe25SConrad Meyer ZSTD_frameParameters fParams, 17989cbefe25SConrad Meyer unsigned long long pledgedSrcSize); 17990c16b537SWarner Losh 18000c16b537SWarner Losh /*! ZSTD_resetCStream() : 18012b9c00cbSConrad Meyer * This function is deprecated, and is equivalent to: 18022b9c00cbSConrad Meyer * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 18032b9c00cbSConrad Meyer * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); 18042b9c00cbSConrad Meyer * 1805a0483764SConrad Meyer * start a new frame, using same parameters from previous frame. 18060f743729SConrad Meyer * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. 18070c16b537SWarner Losh * Note that zcs must be init at least once before using ZSTD_resetCStream(). 1808052d3c12SConrad Meyer * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. 18090c16b537SWarner Losh * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. 1810052d3c12SConrad Meyer * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, 181119fcbaf1SConrad Meyer * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. 18120f743729SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()) 18139cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 18140f743729SConrad Meyer */ 18150c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); 18160c16b537SWarner Losh 18170c16b537SWarner Losh 181819fcbaf1SConrad Meyer typedef struct { 18190f743729SConrad Meyer unsigned long long ingested; /* nb input bytes read and buffered */ 18200f743729SConrad Meyer unsigned long long consumed; /* nb input bytes actually compressed */ 18210f743729SConrad Meyer unsigned long long produced; /* nb of compressed bytes generated and buffered */ 18220f743729SConrad Meyer unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ 18230f743729SConrad Meyer unsigned currentJobID; /* MT only : latest started job nb */ 18240f743729SConrad Meyer unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ 182519fcbaf1SConrad Meyer } ZSTD_frameProgression; 182619fcbaf1SConrad Meyer 182719fcbaf1SConrad Meyer /* ZSTD_getFrameProgression() : 182819fcbaf1SConrad Meyer * tells how much data has been ingested (read from input) 182919fcbaf1SConrad Meyer * consumed (input actually compressed) and produced (output) for current frame. 18300f743729SConrad Meyer * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. 18310f743729SConrad Meyer * Aggregates progression inside active worker threads. 183219fcbaf1SConrad Meyer */ 18330f743729SConrad Meyer ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); 18340f743729SConrad Meyer 18350f743729SConrad Meyer /*! ZSTD_toFlushNow() : 18360f743729SConrad Meyer * Tell how many bytes are ready to be flushed immediately. 18370f743729SConrad Meyer * Useful for multithreading scenarios (nbWorkers >= 1). 18380f743729SConrad Meyer * Probe the oldest active job, defined as oldest job not yet entirely flushed, 18390f743729SConrad Meyer * and check its output buffer. 18400f743729SConrad Meyer * @return : amount of data stored in oldest job and ready to be flushed immediately. 18410f743729SConrad Meyer * if @return == 0, it means either : 18420f743729SConrad Meyer * + there is no active job (could be checked with ZSTD_frameProgression()), or 18430f743729SConrad Meyer * + oldest job is still actively compressing data, 18440f743729SConrad Meyer * but everything it has produced has also been flushed so far, 1845a0483764SConrad Meyer * therefore flush speed is limited by production speed of oldest job 1846a0483764SConrad Meyer * irrespective of the speed of concurrent (and newer) jobs. 18470f743729SConrad Meyer */ 18480f743729SConrad Meyer ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); 184919fcbaf1SConrad Meyer 185019fcbaf1SConrad Meyer 18510c16b537SWarner Losh /*===== Advanced Streaming decompression functions =====*/ 18522b9c00cbSConrad Meyer /** 18532b9c00cbSConrad Meyer * This function is deprecated, and is equivalent to: 18542b9c00cbSConrad Meyer * 18552b9c00cbSConrad Meyer * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); 18562b9c00cbSConrad Meyer * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); 18572b9c00cbSConrad Meyer * 18582b9c00cbSConrad Meyer * note: no dictionary will be used if dict == NULL or dictSize < 8 18599cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 18602b9c00cbSConrad Meyer */ 18612b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); 18629cbefe25SConrad Meyer 18632b9c00cbSConrad Meyer /** 18642b9c00cbSConrad Meyer * This function is deprecated, and is equivalent to: 18652b9c00cbSConrad Meyer * 18662b9c00cbSConrad Meyer * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); 18672b9c00cbSConrad Meyer * ZSTD_DCtx_refDDict(zds, ddict); 18682b9c00cbSConrad Meyer * 18692b9c00cbSConrad Meyer * note : ddict is referenced, it must outlive decompression session 18709cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 18712b9c00cbSConrad Meyer */ 18722b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); 18739cbefe25SConrad Meyer 18742b9c00cbSConrad Meyer /** 18752b9c00cbSConrad Meyer * This function is deprecated, and is equivalent to: 18762b9c00cbSConrad Meyer * 18772b9c00cbSConrad Meyer * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); 18782b9c00cbSConrad Meyer * 18792b9c00cbSConrad Meyer * re-use decompression parameters from previous init; saves dictionary loading 18809cbefe25SConrad Meyer * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x 18812b9c00cbSConrad Meyer */ 18822b9c00cbSConrad Meyer ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); 18830c16b537SWarner Losh 18840c16b537SWarner Losh 18850c16b537SWarner Losh /********************************************************************* 18860c16b537SWarner Losh * Buffer-less and synchronous inner streaming functions 18870c16b537SWarner Losh * 18880c16b537SWarner Losh * This is an advanced API, giving full control over buffer management, for users which need direct control over memory. 18890c16b537SWarner Losh * But it's also a complex one, with several restrictions, documented below. 18900c16b537SWarner Losh * Prefer normal streaming API for an easier experience. 18910c16b537SWarner Losh ********************************************************************* */ 18920c16b537SWarner Losh 18930c16b537SWarner Losh /** 18940c16b537SWarner Losh Buffer-less streaming compression (synchronous mode) 18950c16b537SWarner Losh 18960c16b537SWarner Losh A ZSTD_CCtx object is required to track streaming operations. 18970c16b537SWarner Losh Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. 18980c16b537SWarner Losh ZSTD_CCtx object can be re-used multiple times within successive compression operations. 18990c16b537SWarner Losh 19000c16b537SWarner Losh Start by initializing a context. 19010c16b537SWarner Losh Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, 19020c16b537SWarner Losh or ZSTD_compressBegin_advanced(), for finer parameter control. 19030c16b537SWarner Losh It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() 19040c16b537SWarner Losh 19050c16b537SWarner Losh Then, consume your input using ZSTD_compressContinue(). 19060c16b537SWarner Losh There are some important considerations to keep in mind when using this advanced function : 19070c16b537SWarner Losh - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. 19080c16b537SWarner Losh - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. 19090c16b537SWarner Losh - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. 19100c16b537SWarner Losh Worst case evaluation is provided by ZSTD_compressBound(). 19110c16b537SWarner Losh ZSTD_compressContinue() doesn't guarantee recover after a failed compression. 19120c16b537SWarner Losh - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). 19130c16b537SWarner Losh It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) 19140c16b537SWarner Losh - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. 19150c16b537SWarner Losh In which case, it will "discard" the relevant memory section from its history. 19160c16b537SWarner Losh 19170c16b537SWarner Losh Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. 19180c16b537SWarner Losh It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. 19190c16b537SWarner Losh Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. 19200c16b537SWarner Losh 19210c16b537SWarner Losh `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. 19220c16b537SWarner Losh */ 19230c16b537SWarner Losh 19240c16b537SWarner Losh /*===== Buffer-less streaming compression functions =====*/ 19250c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); 19260c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); 1927052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ 19280c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ 1929052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ 1930052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ 19310c16b537SWarner Losh 19320c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 19330c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 19340c16b537SWarner Losh 19350c16b537SWarner Losh 19360c16b537SWarner Losh /*- 19370c16b537SWarner Losh Buffer-less streaming decompression (synchronous mode) 19380c16b537SWarner Losh 19390c16b537SWarner Losh A ZSTD_DCtx object is required to track streaming operations. 19400c16b537SWarner Losh Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. 19410c16b537SWarner Losh A ZSTD_DCtx object can be re-used multiple times. 19420c16b537SWarner Losh 19430c16b537SWarner Losh First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). 19440c16b537SWarner Losh Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. 19450c16b537SWarner Losh Data fragment must be large enough to ensure successful decoding. 19460c16b537SWarner Losh `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. 19470c16b537SWarner Losh @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. 19480c16b537SWarner Losh >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. 19490c16b537SWarner Losh errorCode, which can be tested using ZSTD_isError(). 19500c16b537SWarner Losh 19510c16b537SWarner Losh It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, 19520c16b537SWarner Losh such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). 19530c16b537SWarner Losh Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. 19540c16b537SWarner Losh As a consequence, check that values remain within valid application range. 19550c16b537SWarner Losh For example, do not allocate memory blindly, check that `windowSize` is within expectation. 19560c16b537SWarner Losh Each application can set its own limits, depending on local restrictions. 19570c16b537SWarner Losh For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. 19580c16b537SWarner Losh 19590c16b537SWarner Losh ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. 19600c16b537SWarner Losh ZSTD_decompressContinue() is very sensitive to contiguity, 19610c16b537SWarner Losh if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, 19620c16b537SWarner Losh or that previous contiguous segment is large enough to properly handle maximum back-reference distance. 19630c16b537SWarner Losh There are multiple ways to guarantee this condition. 19640c16b537SWarner Losh 19650c16b537SWarner Losh The most memory efficient way is to use a round buffer of sufficient size. 19660c16b537SWarner Losh Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), 19670c16b537SWarner Losh which can @return an error code if required value is too large for current system (in 32-bits mode). 19680c16b537SWarner Losh In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, 19690c16b537SWarner Losh up to the moment there is not enough room left in the buffer to guarantee decoding another full block, 19700c16b537SWarner Losh which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. 19710c16b537SWarner Losh At which point, decoding can resume from the beginning of the buffer. 19720c16b537SWarner Losh Note that already decoded data stored in the buffer should be flushed before being overwritten. 19730c16b537SWarner Losh 19740c16b537SWarner Losh There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. 19750c16b537SWarner Losh 19760c16b537SWarner Losh Finally, if you control the compression process, you can also ignore all buffer size rules, 19770c16b537SWarner Losh as long as the encoder and decoder progress in "lock-step", 19780c16b537SWarner Losh aka use exactly the same buffer sizes, break contiguity at the same place, etc. 19790c16b537SWarner Losh 19800c16b537SWarner Losh Once buffers are setup, start decompression, with ZSTD_decompressBegin(). 19810c16b537SWarner Losh If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). 19820c16b537SWarner Losh 19830c16b537SWarner Losh Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. 19840c16b537SWarner Losh ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). 19850c16b537SWarner Losh ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. 19860c16b537SWarner Losh 19870c16b537SWarner Losh @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). 19880c16b537SWarner Losh It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. 19890c16b537SWarner Losh It can also be an error code, which can be tested with ZSTD_isError(). 19900c16b537SWarner Losh 19910c16b537SWarner Losh A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. 19920c16b537SWarner Losh Context can then be reset to start a new decompression. 19930c16b537SWarner Losh 19940c16b537SWarner Losh Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). 19950c16b537SWarner Losh This information is not required to properly decode a frame. 19960c16b537SWarner Losh 19970c16b537SWarner Losh == Special case : skippable frames == 19980c16b537SWarner Losh 19990c16b537SWarner Losh Skippable frames allow integration of user-defined data into a flow of concatenated frames. 20000c16b537SWarner Losh Skippable frames will be ignored (skipped) by decompressor. 20010c16b537SWarner Losh The format of skippable frames is as follows : 20020c16b537SWarner Losh a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F 20030c16b537SWarner Losh b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits 20040c16b537SWarner Losh c) Frame Content - any content (User Data) of length equal to Frame Size 20050c16b537SWarner Losh For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. 20060c16b537SWarner Losh For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. 20070c16b537SWarner Losh */ 20080c16b537SWarner Losh 20090c16b537SWarner Losh /*===== Buffer-less streaming decompression functions =====*/ 20100c16b537SWarner Losh typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; 20110c16b537SWarner Losh typedef struct { 20120c16b537SWarner Losh unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ 20130c16b537SWarner Losh unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ 20140c16b537SWarner Losh unsigned blockSizeMax; 20150c16b537SWarner Losh ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ 20160c16b537SWarner Losh unsigned headerSize; 20170c16b537SWarner Losh unsigned dictID; 20180c16b537SWarner Losh unsigned checksumFlag; 20190c16b537SWarner Losh } ZSTD_frameHeader; 2020a0483764SConrad Meyer 20214d3f1eafSConrad Meyer /*! ZSTD_getFrameHeader() : 20220f743729SConrad Meyer * decode Frame Header, or requires larger `srcSize`. 20230f743729SConrad Meyer * @return : 0, `zfhPtr` is correctly filled, 20240f743729SConrad Meyer * >0, `srcSize` is too small, value is wanted `srcSize` amount, 20250f743729SConrad Meyer * or an error code, which can be tested using ZSTD_isError() */ 20260c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ 2027a0483764SConrad Meyer /*! ZSTD_getFrameHeader_advanced() : 2028a0483764SConrad Meyer * same as ZSTD_getFrameHeader(), 2029a0483764SConrad Meyer * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ 2030a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); 20310c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ 20320c16b537SWarner Losh 20330c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); 20340c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); 20350c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 20360c16b537SWarner Losh 20370c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); 20380c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 20390c16b537SWarner Losh 20400c16b537SWarner Losh /* misc */ 20410c16b537SWarner Losh ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); 20420c16b537SWarner Losh typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; 20430c16b537SWarner Losh ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); 20440c16b537SWarner Losh 20450c16b537SWarner Losh 20460c16b537SWarner Losh 20470c16b537SWarner Losh 20480c16b537SWarner Losh /* ============================ */ 20490c16b537SWarner Losh /** Block level API */ 20500c16b537SWarner Losh /* ============================ */ 20510c16b537SWarner Losh 20520c16b537SWarner Losh /*! 20530c16b537SWarner Losh Block functions produce and decode raw zstd blocks, without frame metadata. 20549cbefe25SConrad Meyer Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). 20559cbefe25SConrad Meyer But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. 20560c16b537SWarner Losh 20570c16b537SWarner Losh A few rules to respect : 20580c16b537SWarner Losh - Compressing and decompressing require a context structure 20590c16b537SWarner Losh + Use ZSTD_createCCtx() and ZSTD_createDCtx() 20600c16b537SWarner Losh - It is necessary to init context before starting 20610c16b537SWarner Losh + compression : any ZSTD_compressBegin*() variant, including with dictionary 20620c16b537SWarner Losh + decompression : any ZSTD_decompressBegin*() variant, including with dictionary 20630c16b537SWarner Losh + copyCCtx() and copyDCtx() can be used too 20640c16b537SWarner Losh - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB 20650c16b537SWarner Losh + If input is larger than a block size, it's necessary to split input data into multiple blocks 20669cbefe25SConrad Meyer + For inputs larger than a single block, consider using regular ZSTD_compress() instead. 20679cbefe25SConrad Meyer Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. 20689cbefe25SConrad Meyer - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! 20699cbefe25SConrad Meyer ===> In which case, nothing is produced into `dst` ! 20709cbefe25SConrad Meyer + User __must__ test for such outcome and deal directly with uncompressed data 20719cbefe25SConrad Meyer + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. 20729cbefe25SConrad Meyer Doing so would mess up with statistics history, leading to potential data corruption. 20739cbefe25SConrad Meyer + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! 20740c16b537SWarner Losh + In case of multiple successive blocks, should some of them be uncompressed, 20750c16b537SWarner Losh decoder must be informed of their existence in order to follow proper history. 20760c16b537SWarner Losh Use ZSTD_insertBlock() for such a case. 20770c16b537SWarner Losh */ 20780c16b537SWarner Losh 20790c16b537SWarner Losh /*===== Raw zstd block functions =====*/ 20800c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); 20810c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 20820c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 208319fcbaf1SConrad Meyer ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ 20840c16b537SWarner Losh 20850c16b537SWarner Losh 20860c16b537SWarner Losh #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ 20870c16b537SWarner Losh 20880c16b537SWarner Losh #if defined (__cplusplus) 20890c16b537SWarner Losh } 20900c16b537SWarner Losh #endif 2091