10c16b537SWarner Losh /* 20c16b537SWarner Losh * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 30c16b537SWarner Losh * All rights reserved. 40c16b537SWarner Losh * 50c16b537SWarner Losh * This source code is licensed under both the BSD-style license (found in the 60c16b537SWarner Losh * LICENSE file in the root directory of this source tree) and the GPLv2 (found 70c16b537SWarner Losh * in the COPYING file in the root directory of this source tree). 80c16b537SWarner Losh * You may select, at your option, one of the above-listed licenses. 90c16b537SWarner Losh */ 100c16b537SWarner Losh #if defined (__cplusplus) 110c16b537SWarner Losh extern "C" { 120c16b537SWarner Losh #endif 130c16b537SWarner Losh 140c16b537SWarner Losh #ifndef ZSTD_H_235446 150c16b537SWarner Losh #define ZSTD_H_235446 160c16b537SWarner Losh 170c16b537SWarner Losh /* ====== Dependency ======*/ 180c16b537SWarner Losh #include <stddef.h> /* size_t */ 190c16b537SWarner Losh 200c16b537SWarner Losh 210c16b537SWarner Losh /* ===== ZSTDLIB_API : control library symbols visibility ===== */ 220c16b537SWarner Losh #ifndef ZSTDLIB_VISIBILITY 230c16b537SWarner Losh # if defined(__GNUC__) && (__GNUC__ >= 4) 240c16b537SWarner Losh # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) 250c16b537SWarner Losh # else 260c16b537SWarner Losh # define ZSTDLIB_VISIBILITY 270c16b537SWarner Losh # endif 280c16b537SWarner Losh #endif 290c16b537SWarner Losh #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) 300c16b537SWarner Losh # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY 310c16b537SWarner Losh #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) 320c16b537SWarner Losh # define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 330c16b537SWarner Losh #else 340c16b537SWarner Losh # define ZSTDLIB_API ZSTDLIB_VISIBILITY 350c16b537SWarner Losh #endif 360c16b537SWarner Losh 370c16b537SWarner Losh 380f743729SConrad Meyer /******************************************************************************* 390c16b537SWarner Losh Introduction 400c16b537SWarner Losh 410f743729SConrad Meyer zstd, short for Zstandard, is a fast lossless compression algorithm, targeting 420f743729SConrad Meyer real-time compression scenarios at zlib-level and better compression ratios. 430f743729SConrad Meyer The zstd compression library provides in-memory compression and decompression 440f743729SConrad Meyer functions. 450f743729SConrad Meyer 460f743729SConrad Meyer The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), 470f743729SConrad Meyer which is currently 22. Levels >= 20, labeled `--ultra`, should be used with 480f743729SConrad Meyer caution, as they require more memory. The library also offers negative 490f743729SConrad Meyer compression levels, which extend the range of speed vs. ratio preferences. 500f743729SConrad Meyer The lower the level, the faster the speed (at the cost of compression). 510f743729SConrad Meyer 520c16b537SWarner Losh Compression can be done in: 530c16b537SWarner Losh - a single step (described as Simple API) 5419fcbaf1SConrad Meyer - a single step, reusing a context (described as Explicit context) 550c16b537SWarner Losh - unbounded multiple steps (described as Streaming compression) 560c16b537SWarner Losh 570f743729SConrad Meyer The compression ratio achievable on small data can be highly improved using 580f743729SConrad Meyer a dictionary. Dictionary compression can be performed in: 590f743729SConrad Meyer - a single step (described as Simple dictionary API) 600f743729SConrad Meyer - a single step, reusing a dictionary (described as Bulk-processing 610f743729SConrad Meyer dictionary API) 620f743729SConrad Meyer 630f743729SConrad Meyer Advanced experimental functions can be accessed using 640f743729SConrad Meyer `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. 650f743729SConrad Meyer 660f743729SConrad Meyer Advanced experimental APIs should never be used with a dynamically-linked 670f743729SConrad Meyer library. They are not "stable"; their definitions or signatures may change in 680f743729SConrad Meyer the future. Only static linking is allowed. 690f743729SConrad Meyer *******************************************************************************/ 700c16b537SWarner Losh 710c16b537SWarner Losh /*------ Version ------*/ 720c16b537SWarner Losh #define ZSTD_VERSION_MAJOR 1 730c16b537SWarner Losh #define ZSTD_VERSION_MINOR 3 74*a0483764SConrad Meyer #define ZSTD_VERSION_RELEASE 8 750c16b537SWarner Losh 760c16b537SWarner Losh #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) 77*a0483764SConrad Meyer ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ 780c16b537SWarner Losh 790c16b537SWarner Losh #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE 800c16b537SWarner Losh #define ZSTD_QUOTE(str) #str 810c16b537SWarner Losh #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) 820c16b537SWarner Losh #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) 83*a0483764SConrad Meyer ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ 840c16b537SWarner Losh 850f743729SConrad Meyer /*************************************** 860f743729SConrad Meyer * Default constant 870f743729SConrad Meyer ***************************************/ 880f743729SConrad Meyer #ifndef ZSTD_CLEVEL_DEFAULT 890f743729SConrad Meyer # define ZSTD_CLEVEL_DEFAULT 3 900f743729SConrad Meyer #endif 910c16b537SWarner Losh 920c16b537SWarner Losh /*************************************** 930c16b537SWarner Losh * Simple API 940c16b537SWarner Losh ***************************************/ 950c16b537SWarner Losh /*! ZSTD_compress() : 960c16b537SWarner Losh * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. 970c16b537SWarner Losh * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. 980c16b537SWarner Losh * @return : compressed size written into `dst` (<= `dstCapacity), 990c16b537SWarner Losh * or an error code if it fails (which can be tested using ZSTD_isError()). */ 1000c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, 1010c16b537SWarner Losh const void* src, size_t srcSize, 1020c16b537SWarner Losh int compressionLevel); 1030c16b537SWarner Losh 1040c16b537SWarner Losh /*! ZSTD_decompress() : 1050c16b537SWarner Losh * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. 1060c16b537SWarner Losh * `dstCapacity` is an upper bound of originalSize to regenerate. 1070c16b537SWarner Losh * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. 1080c16b537SWarner Losh * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), 1090c16b537SWarner Losh * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ 1100c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, 1110c16b537SWarner Losh const void* src, size_t compressedSize); 1120c16b537SWarner Losh 113*a0483764SConrad Meyer /*! ZSTD_getFrameContentSize() : requires v1.3.0+ 1140c16b537SWarner Losh * `src` should point to the start of a ZSTD encoded frame. 1150c16b537SWarner Losh * `srcSize` must be at least as large as the frame header. 1160c16b537SWarner Losh * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. 1170f743729SConrad Meyer * @return : - decompressed size of `src` frame content, if known 1180c16b537SWarner Losh * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined 1190c16b537SWarner Losh * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) 1200c16b537SWarner Losh * note 1 : a 0 return value means the frame is valid but "empty". 1210c16b537SWarner Losh * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. 1220c16b537SWarner Losh * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. 1230c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 1240c16b537SWarner Losh * Optionally, application can rely on some implicit limit, 1250c16b537SWarner Losh * as ZSTD_decompress() only needs an upper bound of decompressed size. 1260c16b537SWarner Losh * (For example, data could be necessarily cut into blocks <= 16 KB). 1270f743729SConrad Meyer * note 3 : decompressed size is always present when compression is completed using single-pass functions, 1280f743729SConrad Meyer * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). 1290c16b537SWarner Losh * note 4 : decompressed size can be very large (64-bits value), 1300c16b537SWarner Losh * potentially larger than what local system can handle as a single memory segment. 1310c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 1320c16b537SWarner Losh * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. 1330c16b537SWarner Losh * Always ensure return value fits within application's authorized limits. 1340c16b537SWarner Losh * Each application can set its own limits. 1350c16b537SWarner Losh * note 6 : This function replaces ZSTD_getDecompressedSize() */ 1360c16b537SWarner Losh #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) 1370c16b537SWarner Losh #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) 1380c16b537SWarner Losh ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); 1390c16b537SWarner Losh 1400c16b537SWarner Losh /*! ZSTD_getDecompressedSize() : 1410c16b537SWarner Losh * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). 14219fcbaf1SConrad Meyer * Both functions work the same way, but ZSTD_getDecompressedSize() blends 14319fcbaf1SConrad Meyer * "empty", "unknown" and "error" results to the same return value (0), 14419fcbaf1SConrad Meyer * while ZSTD_getFrameContentSize() gives them separate return values. 1450f743729SConrad Meyer * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ 1460c16b537SWarner Losh ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); 1470c16b537SWarner Losh 1480c16b537SWarner Losh 1490c16b537SWarner Losh /*====== Helper functions ======*/ 150052d3c12SConrad Meyer #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ 15119fcbaf1SConrad Meyer ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ 1520c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ 1530c16b537SWarner Losh ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ 1540c16b537SWarner Losh ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ 1550c16b537SWarner Losh 1560c16b537SWarner Losh 1570c16b537SWarner Losh /*************************************** 15819fcbaf1SConrad Meyer * Explicit context 1590c16b537SWarner Losh ***************************************/ 1600c16b537SWarner Losh /*= Compression context 1610c16b537SWarner Losh * When compressing many times, 1620c16b537SWarner Losh * it is recommended to allocate a context just once, and re-use it for each successive compression operation. 1630c16b537SWarner Losh * This will make workload friendlier for system's memory. 1640c16b537SWarner Losh * Use one context per thread for parallel execution in multi-threaded environments. */ 1650c16b537SWarner Losh typedef struct ZSTD_CCtx_s ZSTD_CCtx; 1660c16b537SWarner Losh ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); 1670c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); 1680c16b537SWarner Losh 1690c16b537SWarner Losh /*! ZSTD_compressCCtx() : 170*a0483764SConrad Meyer * Same as ZSTD_compress(), using an explicit ZSTD_CCtx 171*a0483764SConrad Meyer * The function will compress at requested compression level, 172*a0483764SConrad Meyer * ignoring any other parameter */ 173*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, 1740c16b537SWarner Losh void* dst, size_t dstCapacity, 1750c16b537SWarner Losh const void* src, size_t srcSize, 1760c16b537SWarner Losh int compressionLevel); 1770c16b537SWarner Losh 1780c16b537SWarner Losh /*= Decompression context 1790c16b537SWarner Losh * When decompressing many times, 1800c16b537SWarner Losh * it is recommended to allocate a context only once, 1810c16b537SWarner Losh * and re-use it for each successive compression operation. 1820c16b537SWarner Losh * This will make workload friendlier for system's memory. 1830c16b537SWarner Losh * Use one context per thread for parallel execution. */ 1840c16b537SWarner Losh typedef struct ZSTD_DCtx_s ZSTD_DCtx; 1850c16b537SWarner Losh ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); 1860c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); 1870c16b537SWarner Losh 1880c16b537SWarner Losh /*! ZSTD_decompressDCtx() : 189*a0483764SConrad Meyer * Same as ZSTD_decompress(), 190*a0483764SConrad Meyer * requires an allocated ZSTD_DCtx. 191*a0483764SConrad Meyer * Compatible with sticky parameters. 192*a0483764SConrad Meyer */ 193*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, 1940c16b537SWarner Losh void* dst, size_t dstCapacity, 1950c16b537SWarner Losh const void* src, size_t srcSize); 1960c16b537SWarner Losh 1970c16b537SWarner Losh 1980c16b537SWarner Losh /************************** 1990c16b537SWarner Losh * Simple dictionary API 2000c16b537SWarner Losh ***************************/ 2010c16b537SWarner Losh /*! ZSTD_compress_usingDict() : 202*a0483764SConrad Meyer * Compression at an explicit compression level using a Dictionary. 203*a0483764SConrad Meyer * A dictionary can be any arbitrary data segment (also called a prefix), 204*a0483764SConrad Meyer * or a buffer with specified information (see dictBuilder/zdict.h). 2050c16b537SWarner Losh * Note : This function loads the dictionary, resulting in significant startup delay. 206*a0483764SConrad Meyer * It's intended for a dictionary used only once. 207*a0483764SConrad Meyer * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ 2080c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, 2090c16b537SWarner Losh void* dst, size_t dstCapacity, 2100c16b537SWarner Losh const void* src, size_t srcSize, 2110c16b537SWarner Losh const void* dict,size_t dictSize, 2120c16b537SWarner Losh int compressionLevel); 2130c16b537SWarner Losh 2140c16b537SWarner Losh /*! ZSTD_decompress_usingDict() : 215*a0483764SConrad Meyer * Decompression using a known Dictionary. 2160c16b537SWarner Losh * Dictionary must be identical to the one used during compression. 2170c16b537SWarner Losh * Note : This function loads the dictionary, resulting in significant startup delay. 218*a0483764SConrad Meyer * It's intended for a dictionary used only once. 2190c16b537SWarner Losh * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ 2200c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, 2210c16b537SWarner Losh void* dst, size_t dstCapacity, 2220c16b537SWarner Losh const void* src, size_t srcSize, 2230c16b537SWarner Losh const void* dict,size_t dictSize); 2240c16b537SWarner Losh 2250c16b537SWarner Losh 226*a0483764SConrad Meyer /*********************************** 2270c16b537SWarner Losh * Bulk processing dictionary API 228*a0483764SConrad Meyer **********************************/ 2290c16b537SWarner Losh typedef struct ZSTD_CDict_s ZSTD_CDict; 2300c16b537SWarner Losh 2310c16b537SWarner Losh /*! ZSTD_createCDict() : 232*a0483764SConrad Meyer * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. 233*a0483764SConrad Meyer * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. 2340c16b537SWarner Losh * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. 235*a0483764SConrad Meyer * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. 236*a0483764SConrad Meyer * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. 237*a0483764SConrad Meyer * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */ 2380c16b537SWarner Losh ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, 2390c16b537SWarner Losh int compressionLevel); 2400c16b537SWarner Losh 2410c16b537SWarner Losh /*! ZSTD_freeCDict() : 2420c16b537SWarner Losh * Function frees memory allocated by ZSTD_createCDict(). */ 2430c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); 2440c16b537SWarner Losh 2450c16b537SWarner Losh /*! ZSTD_compress_usingCDict() : 2460c16b537SWarner Losh * Compression using a digested Dictionary. 247*a0483764SConrad Meyer * Recommended when same dictionary is used multiple times. 248*a0483764SConrad Meyer * Note : compression level is _decided at dictionary creation time_, 249*a0483764SConrad Meyer * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ 2500c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, 2510c16b537SWarner Losh void* dst, size_t dstCapacity, 2520c16b537SWarner Losh const void* src, size_t srcSize, 2530c16b537SWarner Losh const ZSTD_CDict* cdict); 2540c16b537SWarner Losh 2550c16b537SWarner Losh 2560c16b537SWarner Losh typedef struct ZSTD_DDict_s ZSTD_DDict; 2570c16b537SWarner Losh 2580c16b537SWarner Losh /*! ZSTD_createDDict() : 2590c16b537SWarner Losh * Create a digested dictionary, ready to start decompression operation without startup delay. 260*a0483764SConrad Meyer * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ 2610c16b537SWarner Losh ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); 2620c16b537SWarner Losh 2630c16b537SWarner Losh /*! ZSTD_freeDDict() : 2640c16b537SWarner Losh * Function frees memory allocated with ZSTD_createDDict() */ 2650c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); 2660c16b537SWarner Losh 2670c16b537SWarner Losh /*! ZSTD_decompress_usingDDict() : 2680c16b537SWarner Losh * Decompression using a digested Dictionary. 269*a0483764SConrad Meyer * Recommended when same dictionary is used multiple times. */ 2700c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, 2710c16b537SWarner Losh void* dst, size_t dstCapacity, 2720c16b537SWarner Losh const void* src, size_t srcSize, 2730c16b537SWarner Losh const ZSTD_DDict* ddict); 2740c16b537SWarner Losh 2750c16b537SWarner Losh 2760c16b537SWarner Losh /**************************** 2770c16b537SWarner Losh * Streaming 2780c16b537SWarner Losh ****************************/ 2790c16b537SWarner Losh 2800c16b537SWarner Losh typedef struct ZSTD_inBuffer_s { 2810c16b537SWarner Losh const void* src; /**< start of input buffer */ 2820c16b537SWarner Losh size_t size; /**< size of input buffer */ 2830c16b537SWarner Losh size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ 2840c16b537SWarner Losh } ZSTD_inBuffer; 2850c16b537SWarner Losh 2860c16b537SWarner Losh typedef struct ZSTD_outBuffer_s { 2870c16b537SWarner Losh void* dst; /**< start of output buffer */ 2880c16b537SWarner Losh size_t size; /**< size of output buffer */ 2890c16b537SWarner Losh size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ 2900c16b537SWarner Losh } ZSTD_outBuffer; 2910c16b537SWarner Losh 2920c16b537SWarner Losh 2930c16b537SWarner Losh 2940c16b537SWarner Losh /*-*********************************************************************** 2950c16b537SWarner Losh * Streaming compression - HowTo 2960c16b537SWarner Losh * 2970c16b537SWarner Losh * A ZSTD_CStream object is required to track streaming operation. 2980c16b537SWarner Losh * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. 2990c16b537SWarner Losh * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. 300*a0483764SConrad Meyer * It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. 3010c16b537SWarner Losh * 302*a0483764SConrad Meyer * For parallel execution, use one separate ZSTD_CStream per thread. 303*a0483764SConrad Meyer * 304*a0483764SConrad Meyer * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. 305*a0483764SConrad Meyer * 306*a0483764SConrad Meyer * Parameters are sticky : when starting a new compression on the same context, 307*a0483764SConrad Meyer * it will re-use the same sticky parameters as previous compression session. 308*a0483764SConrad Meyer * When in doubt, it's recommended to fully initialize the context before usage. 309*a0483764SConrad Meyer * Use ZSTD_initCStream() to set the parameter to a selected compression level. 310*a0483764SConrad Meyer * Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters. 3110c16b537SWarner Losh * 3120f743729SConrad Meyer * Use ZSTD_compressStream() as many times as necessary to consume input stream. 3130f743729SConrad Meyer * The function will automatically update both `pos` fields within `input` and `output`. 3140f743729SConrad Meyer * Note that the function may not consume the entire input, 3150f743729SConrad Meyer * for example, because the output buffer is already full, 3160f743729SConrad Meyer * in which case `input.pos < input.size`. 3170f743729SConrad Meyer * The caller must check if input has been entirely consumed. 3180f743729SConrad Meyer * If not, the caller must make some room to receive more compressed data, 3190f743729SConrad Meyer * and then present again remaining input data. 3200c16b537SWarner Losh * @return : a size hint, preferred nb of bytes to use as input for next function call 3210c16b537SWarner Losh * or an error code, which can be tested using ZSTD_isError(). 322*a0483764SConrad Meyer * Note 1 : it's just a hint, to help latency a little, any value will work fine. 3230c16b537SWarner Losh * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() 3240c16b537SWarner Losh * 3250f743729SConrad Meyer * At any moment, it's possible to flush whatever data might remain stuck within internal buffer, 3260f743729SConrad Meyer * using ZSTD_flushStream(). `output->pos` will be updated. 3270f743729SConrad Meyer * Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0). 3280f743729SConrad Meyer * In which case, make some room to receive more compressed data, and call again ZSTD_flushStream(). 3290f743729SConrad Meyer * @return : 0 if internal buffers are entirely flushed, 3300f743729SConrad Meyer * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), 3310c16b537SWarner Losh * or an error code, which can be tested using ZSTD_isError(). 3320c16b537SWarner Losh * 3330c16b537SWarner Losh * ZSTD_endStream() instructs to finish a frame. 3340c16b537SWarner Losh * It will perform a flush and write frame epilogue. 3350c16b537SWarner Losh * The epilogue is required for decoders to consider a frame completed. 3360f743729SConrad Meyer * flush() operation is the same, and follows same rules as ZSTD_flushStream(). 3370c16b537SWarner Losh * @return : 0 if frame fully completed and fully flushed, 3380f743729SConrad Meyer * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), 3390c16b537SWarner Losh * or an error code, which can be tested using ZSTD_isError(). 3400c16b537SWarner Losh * 3410c16b537SWarner Losh * *******************************************************************/ 3420c16b537SWarner Losh 3430c16b537SWarner Losh typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ 3440f743729SConrad Meyer /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ 3450c16b537SWarner Losh /*===== ZSTD_CStream management functions =====*/ 3460c16b537SWarner Losh ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); 3470c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); 3480c16b537SWarner Losh 3490c16b537SWarner Losh /*===== Streaming compression functions =====*/ 3500c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); 3510c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); 3520c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); 3530c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); 3540c16b537SWarner Losh 3550c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ 3560c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ 3570c16b537SWarner Losh 3580c16b537SWarner Losh 3590c16b537SWarner Losh 3600c16b537SWarner Losh /*-*************************************************************************** 3610c16b537SWarner Losh * Streaming decompression - HowTo 3620c16b537SWarner Losh * 3630c16b537SWarner Losh * A ZSTD_DStream object is required to track streaming operations. 3640c16b537SWarner Losh * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. 3650c16b537SWarner Losh * ZSTD_DStream objects can be re-used multiple times. 3660c16b537SWarner Losh * 367*a0483764SConrad Meyer * Use ZSTD_initDStream() to start a new decompression operation. 3680c16b537SWarner Losh * @return : recommended first input size 369*a0483764SConrad Meyer * Alternatively, use advanced API to set specific properties. 3700c16b537SWarner Losh * 3710c16b537SWarner Losh * Use ZSTD_decompressStream() repetitively to consume your input. 3720c16b537SWarner Losh * The function will update both `pos` fields. 3730c16b537SWarner Losh * If `input.pos < input.size`, some input has not been consumed. 3740c16b537SWarner Losh * It's up to the caller to present again remaining data. 375*a0483764SConrad Meyer * The function tries to flush all data decoded immediately, respecting output buffer size. 3760c16b537SWarner Losh * If `output.pos < output.size`, decoder has flushed everything it could. 377*a0483764SConrad Meyer * But if `output.pos == output.size`, there might be some data left within internal buffers., 3780f743729SConrad Meyer * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. 379*a0483764SConrad Meyer * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. 3800c16b537SWarner Losh * @return : 0 when a frame is completely decoded and fully flushed, 3810f743729SConrad Meyer * or an error code, which can be tested using ZSTD_isError(), 3820f743729SConrad Meyer * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : 383*a0483764SConrad Meyer * the return value is a suggested next input size (just a hint for better latency) 384*a0483764SConrad Meyer * that will never request more than the remaining frame size. 3850c16b537SWarner Losh * *******************************************************************************/ 3860c16b537SWarner Losh 3870c16b537SWarner Losh typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ 3880f743729SConrad Meyer /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ 3890c16b537SWarner Losh /*===== ZSTD_DStream management functions =====*/ 3900c16b537SWarner Losh ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); 3910c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); 3920c16b537SWarner Losh 3930c16b537SWarner Losh /*===== Streaming decompression functions =====*/ 3940c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); 3950c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); 3960c16b537SWarner Losh 3970c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ 3980c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ 3990c16b537SWarner Losh 4000c16b537SWarner Losh #endif /* ZSTD_H_235446 */ 4010c16b537SWarner Losh 4020c16b537SWarner Losh 4030c16b537SWarner Losh 4040f743729SConrad Meyer 4050c16b537SWarner Losh /**************************************************************************************** 4060f743729SConrad Meyer * ADVANCED AND EXPERIMENTAL FUNCTIONS 4070f743729SConrad Meyer **************************************************************************************** 408*a0483764SConrad Meyer * The definitions in the following section are considered experimental. 4090c16b537SWarner Losh * They are provided for advanced scenarios. 410*a0483764SConrad Meyer * They should never be used with a dynamic library, as prototypes may change in the future. 4110c16b537SWarner Losh * Use them only in association with static linking. 4120c16b537SWarner Losh * ***************************************************************************************/ 4130c16b537SWarner Losh 414*a0483764SConrad Meyer #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) 415*a0483764SConrad Meyer #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY 4160c16b537SWarner Losh 417*a0483764SConrad Meyer 418*a0483764SConrad Meyer /**************************************************************************************** 419*a0483764SConrad Meyer * Candidate API for promotion to stable status 420*a0483764SConrad Meyer **************************************************************************************** 421*a0483764SConrad Meyer * The following symbols and constants form the "staging area" : 422*a0483764SConrad Meyer * they are considered to join "stable API" by v1.4.0. 423*a0483764SConrad Meyer * The proposal is written so that it can be made stable "as is", 424*a0483764SConrad Meyer * though it's still possible to suggest improvements. 425*a0483764SConrad Meyer * Staging is in fact last chance for changes, 426*a0483764SConrad Meyer * the API is locked once reaching "stable" status. 427*a0483764SConrad Meyer * ***************************************************************************************/ 428*a0483764SConrad Meyer 429*a0483764SConrad Meyer 430*a0483764SConrad Meyer /* === Constants === */ 431*a0483764SConrad Meyer 432*a0483764SConrad Meyer /* all magic numbers are supposed read/written to/from files/memory using little-endian convention */ 433*a0483764SConrad Meyer #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ 434*a0483764SConrad Meyer #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ 435*a0483764SConrad Meyer #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ 436*a0483764SConrad Meyer #define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 4370f743729SConrad Meyer 4380f743729SConrad Meyer #define ZSTD_BLOCKSIZELOG_MAX 17 439*a0483764SConrad Meyer #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) 4400c16b537SWarner Losh 441*a0483764SConrad Meyer 442*a0483764SConrad Meyer /* === query limits === */ 443*a0483764SConrad Meyer 444*a0483764SConrad Meyer ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ 445*a0483764SConrad Meyer 446*a0483764SConrad Meyer 447*a0483764SConrad Meyer /* === frame size === */ 448*a0483764SConrad Meyer 449*a0483764SConrad Meyer /*! ZSTD_findFrameCompressedSize() : 450*a0483764SConrad Meyer * `src` should point to the start of a ZSTD frame or skippable frame. 451*a0483764SConrad Meyer * `srcSize` must be >= first frame size 452*a0483764SConrad Meyer * @return : the compressed size of the first frame starting at `src`, 453*a0483764SConrad Meyer * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, 454*a0483764SConrad Meyer * or an error code if input is invalid */ 455*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); 456*a0483764SConrad Meyer 457*a0483764SConrad Meyer 458*a0483764SConrad Meyer /* === Memory management === */ 459*a0483764SConrad Meyer 460*a0483764SConrad Meyer /*! ZSTD_sizeof_*() : 461*a0483764SConrad Meyer * These functions give the _current_ memory usage of selected object. 462*a0483764SConrad Meyer * Note that object memory usage can evolve (increase or decrease) over time. */ 463*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); 464*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); 465*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); 466*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); 467*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); 468*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); 469*a0483764SConrad Meyer 470*a0483764SConrad Meyer 471*a0483764SConrad Meyer /*************************************** 472*a0483764SConrad Meyer * Advanced compression API 473*a0483764SConrad Meyer ***************************************/ 474*a0483764SConrad Meyer 475*a0483764SConrad Meyer /* API design : 476*a0483764SConrad Meyer * Parameters are pushed one by one into an existing context, 477*a0483764SConrad Meyer * using ZSTD_CCtx_set*() functions. 478*a0483764SConrad Meyer * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. 479*a0483764SConrad Meyer * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! 480*a0483764SConrad Meyer * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx() 481*a0483764SConrad Meyer * 482*a0483764SConrad Meyer * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). 483*a0483764SConrad Meyer * 484*a0483764SConrad Meyer * This API supercedes all other "advanced" API entry points in the experimental section. 485*a0483764SConrad Meyer * In the future, we expect to remove from experimental API entry points which are redundant with this API. 486*a0483764SConrad Meyer */ 487*a0483764SConrad Meyer 488*a0483764SConrad Meyer 489*a0483764SConrad Meyer /* Compression strategies, listed from fastest to strongest */ 490*a0483764SConrad Meyer typedef enum { ZSTD_fast=1, 491*a0483764SConrad Meyer ZSTD_dfast=2, 492*a0483764SConrad Meyer ZSTD_greedy=3, 493*a0483764SConrad Meyer ZSTD_lazy=4, 494*a0483764SConrad Meyer ZSTD_lazy2=5, 495*a0483764SConrad Meyer ZSTD_btlazy2=6, 496*a0483764SConrad Meyer ZSTD_btopt=7, 497*a0483764SConrad Meyer ZSTD_btultra=8, 498*a0483764SConrad Meyer ZSTD_btultra2=9 499*a0483764SConrad Meyer /* note : new strategies _might_ be added in the future. 500*a0483764SConrad Meyer Only the order (from fast to strong) is guaranteed */ 501*a0483764SConrad Meyer } ZSTD_strategy; 502*a0483764SConrad Meyer 503*a0483764SConrad Meyer 504*a0483764SConrad Meyer typedef enum { 505*a0483764SConrad Meyer 506*a0483764SConrad Meyer /* compression parameters */ 507*a0483764SConrad Meyer ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table 508*a0483764SConrad Meyer * Default level is ZSTD_CLEVEL_DEFAULT==3. 509*a0483764SConrad Meyer * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. 510*a0483764SConrad Meyer * Note 1 : it's possible to pass a negative compression level. 511*a0483764SConrad Meyer * Note 2 : setting a level sets all default values of other compression parameters */ 512*a0483764SConrad Meyer ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. 513*a0483764SConrad Meyer * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. 514*a0483764SConrad Meyer * Special: value 0 means "use default windowLog". 515*a0483764SConrad Meyer * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT 516*a0483764SConrad Meyer * requires explicitly allowing such window size at decompression stage if using streaming. */ 517*a0483764SConrad Meyer ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. 518*a0483764SConrad Meyer * Resulting memory usage is (1 << (hashLog+2)). 519*a0483764SConrad Meyer * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. 520*a0483764SConrad Meyer * Larger tables improve compression ratio of strategies <= dFast, 521*a0483764SConrad Meyer * and improve speed of strategies > dFast. 522*a0483764SConrad Meyer * Special: value 0 means "use default hashLog". */ 523*a0483764SConrad Meyer ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. 524*a0483764SConrad Meyer * Resulting memory usage is (1 << (chainLog+2)). 525*a0483764SConrad Meyer * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. 526*a0483764SConrad Meyer * Larger tables result in better and slower compression. 527*a0483764SConrad Meyer * This parameter is useless when using "fast" strategy. 528*a0483764SConrad Meyer * It's still useful when using "dfast" strategy, 529*a0483764SConrad Meyer * in which case it defines a secondary probe table. 530*a0483764SConrad Meyer * Special: value 0 means "use default chainLog". */ 531*a0483764SConrad Meyer ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. 532*a0483764SConrad Meyer * More attempts result in better and slower compression. 533*a0483764SConrad Meyer * This parameter is useless when using "fast" and "dFast" strategies. 534*a0483764SConrad Meyer * Special: value 0 means "use default searchLog". */ 535*a0483764SConrad Meyer ZSTD_c_minMatch=105, /* Minimum size of searched matches. 536*a0483764SConrad Meyer * Note that Zstandard can still find matches of smaller size, 537*a0483764SConrad Meyer * it just tweaks its search algorithm to look for this size and larger. 538*a0483764SConrad Meyer * Larger values increase compression and decompression speed, but decrease ratio. 539*a0483764SConrad Meyer * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. 540*a0483764SConrad Meyer * Note that currently, for all strategies < btopt, effective minimum is 4. 541*a0483764SConrad Meyer * , for all strategies > fast, effective maximum is 6. 542*a0483764SConrad Meyer * Special: value 0 means "use default minMatchLength". */ 543*a0483764SConrad Meyer ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. 544*a0483764SConrad Meyer * For strategies btopt, btultra & btultra2: 545*a0483764SConrad Meyer * Length of Match considered "good enough" to stop search. 546*a0483764SConrad Meyer * Larger values make compression stronger, and slower. 547*a0483764SConrad Meyer * For strategy fast: 548*a0483764SConrad Meyer * Distance between match sampling. 549*a0483764SConrad Meyer * Larger values make compression faster, and weaker. 550*a0483764SConrad Meyer * Special: value 0 means "use default targetLength". */ 551*a0483764SConrad Meyer ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. 552*a0483764SConrad Meyer * The higher the value of selected strategy, the more complex it is, 553*a0483764SConrad Meyer * resulting in stronger and slower compression. 554*a0483764SConrad Meyer * Special: value 0 means "use default strategy". */ 555*a0483764SConrad Meyer 556*a0483764SConrad Meyer /* LDM mode parameters */ 557*a0483764SConrad Meyer ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. 558*a0483764SConrad Meyer * This parameter is designed to improve compression ratio 559*a0483764SConrad Meyer * for large inputs, by finding large matches at long distance. 560*a0483764SConrad Meyer * It increases memory usage and window size. 561*a0483764SConrad Meyer * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB 562*a0483764SConrad Meyer * except when expressly set to a different value. */ 563*a0483764SConrad Meyer ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. 564*a0483764SConrad Meyer * Larger values increase memory usage and compression ratio, 565*a0483764SConrad Meyer * but decrease compression speed. 566*a0483764SConrad Meyer * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX 567*a0483764SConrad Meyer * default: windowlog - 7. 568*a0483764SConrad Meyer * Special: value 0 means "automatically determine hashlog". */ 569*a0483764SConrad Meyer ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. 570*a0483764SConrad Meyer * Larger/too small values usually decrease compression ratio. 571*a0483764SConrad Meyer * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. 572*a0483764SConrad Meyer * Special: value 0 means "use default value" (default: 64). */ 573*a0483764SConrad Meyer ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. 574*a0483764SConrad Meyer * Larger values improve collision resolution but decrease compression speed. 575*a0483764SConrad Meyer * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. 576*a0483764SConrad Meyer * Special: value 0 means "use default value" (default: 3). */ 577*a0483764SConrad Meyer ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. 578*a0483764SConrad Meyer * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). 579*a0483764SConrad Meyer * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. 580*a0483764SConrad Meyer * Larger values improve compression speed. 581*a0483764SConrad Meyer * Deviating far from default value will likely result in a compression ratio decrease. 582*a0483764SConrad Meyer * Special: value 0 means "automatically determine hashRateLog". */ 583*a0483764SConrad Meyer 584*a0483764SConrad Meyer /* frame parameters */ 585*a0483764SConrad Meyer ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) 586*a0483764SConrad Meyer * Content size must be known at the beginning of compression. 587*a0483764SConrad Meyer * This is automatically the case when using ZSTD_compress2(), 588*a0483764SConrad Meyer * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ 589*a0483764SConrad Meyer ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ 590*a0483764SConrad Meyer ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ 591*a0483764SConrad Meyer 592*a0483764SConrad Meyer /* multi-threading parameters */ 593*a0483764SConrad Meyer /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). 594*a0483764SConrad Meyer * They return an error otherwise. */ 595*a0483764SConrad Meyer ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. 596*a0483764SConrad Meyer * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : 597*a0483764SConrad Meyer * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, 598*a0483764SConrad Meyer * while compression work is performed in parallel, within worker threads. 599*a0483764SConrad Meyer * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : 600*a0483764SConrad Meyer * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). 601*a0483764SConrad Meyer * More workers improve speed, but also increase memory usage. 602*a0483764SConrad Meyer * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ 603*a0483764SConrad Meyer ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. 604*a0483764SConrad Meyer * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. 605*a0483764SConrad Meyer * 0 means default, which is dynamically determined based on compression parameters. 606*a0483764SConrad Meyer * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. 607*a0483764SConrad Meyer * The minimum size is automatically and transparently enforced */ 608*a0483764SConrad Meyer ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. 609*a0483764SConrad Meyer * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. 610*a0483764SConrad Meyer * It helps preserve compression ratio, while each job is compressed in parallel. 611*a0483764SConrad Meyer * This value is enforced only when nbWorkers >= 1. 612*a0483764SConrad Meyer * Larger values increase compression ratio, but decrease speed. 613*a0483764SConrad Meyer * Possible values range from 0 to 9 : 614*a0483764SConrad Meyer * - 0 means "default" : value will be determined by the library, depending on strategy 615*a0483764SConrad Meyer * - 1 means "no overlap" 616*a0483764SConrad Meyer * - 9 means "full overlap", using a full window size. 617*a0483764SConrad Meyer * Each intermediate rank increases/decreases load size by a factor 2 : 618*a0483764SConrad Meyer * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default 619*a0483764SConrad Meyer * default value varies between 6 and 9, depending on strategy */ 620*a0483764SConrad Meyer 621*a0483764SConrad Meyer /* note : additional experimental parameters are also available 622*a0483764SConrad Meyer * within the experimental section of the API. 623*a0483764SConrad Meyer * At the time of this writing, they include : 624*a0483764SConrad Meyer * ZSTD_c_rsyncable 625*a0483764SConrad Meyer * ZSTD_c_format 626*a0483764SConrad Meyer * ZSTD_c_forceMaxWindow 627*a0483764SConrad Meyer * ZSTD_c_forceAttachDict 628*a0483764SConrad Meyer * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. 629*a0483764SConrad Meyer * note : never ever use experimentalParam? names directly; 630*a0483764SConrad Meyer * also, the enums values themselves are unstable and can still change. 631*a0483764SConrad Meyer */ 632*a0483764SConrad Meyer ZSTD_c_experimentalParam1=500, 633*a0483764SConrad Meyer ZSTD_c_experimentalParam2=10, 634*a0483764SConrad Meyer ZSTD_c_experimentalParam3=1000, 635*a0483764SConrad Meyer ZSTD_c_experimentalParam4=1001 636*a0483764SConrad Meyer } ZSTD_cParameter; 637*a0483764SConrad Meyer 638*a0483764SConrad Meyer 639*a0483764SConrad Meyer typedef struct { 640*a0483764SConrad Meyer size_t error; 641*a0483764SConrad Meyer int lowerBound; 642*a0483764SConrad Meyer int upperBound; 643*a0483764SConrad Meyer } ZSTD_bounds; 644*a0483764SConrad Meyer 645*a0483764SConrad Meyer /*! ZSTD_cParam_getBounds() : 646*a0483764SConrad Meyer * All parameters must belong to an interval with lower and upper bounds, 647*a0483764SConrad Meyer * otherwise they will either trigger an error or be automatically clamped. 648*a0483764SConrad Meyer * @return : a structure, ZSTD_bounds, which contains 649*a0483764SConrad Meyer * - an error status field, which must be tested using ZSTD_isError() 650*a0483764SConrad Meyer * - lower and upper bounds, both inclusive 651*a0483764SConrad Meyer */ 652*a0483764SConrad Meyer ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); 653*a0483764SConrad Meyer 654*a0483764SConrad Meyer /*! ZSTD_CCtx_setParameter() : 655*a0483764SConrad Meyer * Set one compression parameter, selected by enum ZSTD_cParameter. 656*a0483764SConrad Meyer * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). 657*a0483764SConrad Meyer * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). 658*a0483764SConrad Meyer * Setting a parameter is generally only possible during frame initialization (before starting compression). 659*a0483764SConrad Meyer * Exception : when using multi-threading mode (nbWorkers >= 1), 660*a0483764SConrad Meyer * the following parameters can be updated _during_ compression (within same frame): 661*a0483764SConrad Meyer * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. 662*a0483764SConrad Meyer * new parameters will be active for next job only (after a flush()). 663*a0483764SConrad Meyer * @return : an error code (which can be tested using ZSTD_isError()). 664*a0483764SConrad Meyer */ 665*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); 666*a0483764SConrad Meyer 667*a0483764SConrad Meyer /*! ZSTD_CCtx_setPledgedSrcSize() : 668*a0483764SConrad Meyer * Total input data size to be compressed as a single frame. 669*a0483764SConrad Meyer * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. 670*a0483764SConrad Meyer * This value will also be controlled at end of frame, and trigger an error if not respected. 671*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 672*a0483764SConrad Meyer * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. 673*a0483764SConrad Meyer * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. 674*a0483764SConrad Meyer * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. 675*a0483764SConrad Meyer * Note 2 : pledgedSrcSize is only valid once, for the next frame. 676*a0483764SConrad Meyer * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. 677*a0483764SConrad Meyer * Note 3 : Whenever all input data is provided and consumed in a single round, 678*a0483764SConrad Meyer * for example with ZSTD_compress2(), 679*a0483764SConrad Meyer * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), 680*a0483764SConrad Meyer * this value is automatically overriden by srcSize instead. 681*a0483764SConrad Meyer */ 682*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); 683*a0483764SConrad Meyer 684*a0483764SConrad Meyer /*! ZSTD_CCtx_loadDictionary() : 685*a0483764SConrad Meyer * Create an internal CDict from `dict` buffer. 686*a0483764SConrad Meyer * Decompression will have to use same dictionary. 687*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 688*a0483764SConrad Meyer * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, 689*a0483764SConrad Meyer * meaning "return to no-dictionary mode". 690*a0483764SConrad Meyer * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. 691*a0483764SConrad Meyer * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). 692*a0483764SConrad Meyer * Note 2 : Loading a dictionary involves building tables. 693*a0483764SConrad Meyer * It's also a CPU consuming operation, with non-negligible impact on latency. 694*a0483764SConrad Meyer * Tables are dependent on compression parameters, and for this reason, 695*a0483764SConrad Meyer * compression parameters can no longer be changed after loading a dictionary. 696*a0483764SConrad Meyer * Note 3 :`dict` content will be copied internally. 697*a0483764SConrad Meyer * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. 698*a0483764SConrad Meyer * In such a case, dictionary buffer must outlive its users. 699*a0483764SConrad Meyer * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() 700*a0483764SConrad Meyer * to precisely select how dictionary content must be interpreted. */ 701*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); 702*a0483764SConrad Meyer 703*a0483764SConrad Meyer /*! ZSTD_CCtx_refCDict() : 704*a0483764SConrad Meyer * Reference a prepared dictionary, to be used for all next compressed frames. 705*a0483764SConrad Meyer * Note that compression parameters are enforced from within CDict, 706*a0483764SConrad Meyer * and supercede any compression parameter previously set within CCtx. 707*a0483764SConrad Meyer * The dictionary will remain valid for future compressed frames using same CCtx. 708*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 709*a0483764SConrad Meyer * Special : Referencing a NULL CDict means "return to no-dictionary mode". 710*a0483764SConrad Meyer * Note 1 : Currently, only one dictionary can be managed. 711*a0483764SConrad Meyer * Referencing a new dictionary effectively "discards" any previous one. 712*a0483764SConrad Meyer * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ 713*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); 714*a0483764SConrad Meyer 715*a0483764SConrad Meyer /*! ZSTD_CCtx_refPrefix() : 716*a0483764SConrad Meyer * Reference a prefix (single-usage dictionary) for next compressed frame. 717*a0483764SConrad Meyer * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). 718*a0483764SConrad Meyer * Decompression will need same prefix to properly regenerate data. 719*a0483764SConrad Meyer * Compressing with a prefix is similar in outcome as performing a diff and compressing it, 720*a0483764SConrad Meyer * but performs much faster, especially during decompression (compression speed is tunable with compression level). 721*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 722*a0483764SConrad Meyer * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary 723*a0483764SConrad Meyer * Note 1 : Prefix buffer is referenced. It **must** outlive compression. 724*a0483764SConrad Meyer * Its content must remain unmodified during compression. 725*a0483764SConrad Meyer * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, 726*a0483764SConrad Meyer * ensure that the window size is large enough to contain the entire source. 727*a0483764SConrad Meyer * See ZSTD_c_windowLog. 728*a0483764SConrad Meyer * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. 729*a0483764SConrad Meyer * It's a CPU consuming operation, with non-negligible impact on latency. 730*a0483764SConrad Meyer * If there is a need to use the same prefix multiple times, consider loadDictionary instead. 731*a0483764SConrad Meyer * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). 732*a0483764SConrad Meyer * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ 733*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, 734*a0483764SConrad Meyer const void* prefix, size_t prefixSize); 735*a0483764SConrad Meyer 736*a0483764SConrad Meyer 737*a0483764SConrad Meyer typedef enum { 738*a0483764SConrad Meyer ZSTD_reset_session_only = 1, 739*a0483764SConrad Meyer ZSTD_reset_parameters = 2, 740*a0483764SConrad Meyer ZSTD_reset_session_and_parameters = 3 741*a0483764SConrad Meyer } ZSTD_ResetDirective; 742*a0483764SConrad Meyer 743*a0483764SConrad Meyer /*! ZSTD_CCtx_reset() : 744*a0483764SConrad Meyer * There are 2 different things that can be reset, independently or jointly : 745*a0483764SConrad Meyer * - The session : will stop compressing current frame, and make CCtx ready to start a new one. 746*a0483764SConrad Meyer * Useful after an error, or to interrupt any ongoing compression. 747*a0483764SConrad Meyer * Any internal data not yet flushed is cancelled. 748*a0483764SConrad Meyer * Compression parameters and dictionary remain unchanged. 749*a0483764SConrad Meyer * They will be used to compress next frame. 750*a0483764SConrad Meyer * Resetting session never fails. 751*a0483764SConrad Meyer * - The parameters : changes all parameters back to "default". 752*a0483764SConrad Meyer * This removes any reference to any dictionary too. 753*a0483764SConrad Meyer * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) 754*a0483764SConrad Meyer * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) 755*a0483764SConrad Meyer * - Both : similar to resetting the session, followed by resetting parameters. 756*a0483764SConrad Meyer */ 757*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); 758*a0483764SConrad Meyer 759*a0483764SConrad Meyer 760*a0483764SConrad Meyer 761*a0483764SConrad Meyer /*! ZSTD_compress2() : 762*a0483764SConrad Meyer * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. 763*a0483764SConrad Meyer * ZSTD_compress2() always starts a new frame. 764*a0483764SConrad Meyer * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. 765*a0483764SConrad Meyer * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() 766*a0483764SConrad Meyer * - The function is always blocking, returns when compression is completed. 767*a0483764SConrad Meyer * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. 768*a0483764SConrad Meyer * @return : compressed size written into `dst` (<= `dstCapacity), 769*a0483764SConrad Meyer * or an error code if it fails (which can be tested using ZSTD_isError()). 770*a0483764SConrad Meyer */ 771*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, 772*a0483764SConrad Meyer void* dst, size_t dstCapacity, 773*a0483764SConrad Meyer const void* src, size_t srcSize); 774*a0483764SConrad Meyer 775*a0483764SConrad Meyer typedef enum { 776*a0483764SConrad Meyer ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ 777*a0483764SConrad Meyer ZSTD_e_flush=1, /* flush any data provided so far, 778*a0483764SConrad Meyer * it creates (at least) one new block, that can be decoded immediately on reception; 779*a0483764SConrad Meyer * frame will continue: any future data can still reference previously compressed data, improving compression. */ 780*a0483764SConrad Meyer ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. 781*a0483764SConrad Meyer * note that frame is only closed after compressed data is fully flushed (return value == 0). 782*a0483764SConrad Meyer * After that point, any additional data starts a new frame. 783*a0483764SConrad Meyer * note : each frame is independent (does not reference any content from previous frame). */ 784*a0483764SConrad Meyer } ZSTD_EndDirective; 785*a0483764SConrad Meyer 786*a0483764SConrad Meyer /*! ZSTD_compressStream2() : 787*a0483764SConrad Meyer * Behaves about the same as ZSTD_compressStream, with additional control on end directive. 788*a0483764SConrad Meyer * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() 789*a0483764SConrad Meyer * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) 790*a0483764SConrad Meyer * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize 791*a0483764SConrad Meyer * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. 792*a0483764SConrad Meyer * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. 793*a0483764SConrad Meyer * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, 794*a0483764SConrad Meyer * and then immediately returns, just indicating that there is some data remaining to be flushed. 795*a0483764SConrad Meyer * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. 796*a0483764SConrad Meyer * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. 797*a0483764SConrad Meyer * - @return provides a minimum amount of data remaining to be flushed from internal buffers 798*a0483764SConrad Meyer * or an error code, which can be tested using ZSTD_isError(). 799*a0483764SConrad Meyer * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. 800*a0483764SConrad Meyer * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. 801*a0483764SConrad Meyer * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. 802*a0483764SConrad Meyer * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), 803*a0483764SConrad Meyer * only ZSTD_e_end or ZSTD_e_flush operations are allowed. 804*a0483764SConrad Meyer * Before starting a new compression job, or changing compression parameters, 805*a0483764SConrad Meyer * it is required to fully flush internal buffers. 806*a0483764SConrad Meyer */ 807*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, 808*a0483764SConrad Meyer ZSTD_outBuffer* output, 809*a0483764SConrad Meyer ZSTD_inBuffer* input, 810*a0483764SConrad Meyer ZSTD_EndDirective endOp); 811*a0483764SConrad Meyer 812*a0483764SConrad Meyer 813*a0483764SConrad Meyer 814*a0483764SConrad Meyer /* ============================== */ 815*a0483764SConrad Meyer /* Advanced decompression API */ 816*a0483764SConrad Meyer /* ============================== */ 817*a0483764SConrad Meyer 818*a0483764SConrad Meyer /* The advanced API pushes parameters one by one into an existing DCtx context. 819*a0483764SConrad Meyer * Parameters are sticky, and remain valid for all following frames 820*a0483764SConrad Meyer * using the same DCtx context. 821*a0483764SConrad Meyer * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). 822*a0483764SConrad Meyer * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). 823*a0483764SConrad Meyer * Therefore, no new decompression function is necessary. 824*a0483764SConrad Meyer */ 825*a0483764SConrad Meyer 826*a0483764SConrad Meyer 827*a0483764SConrad Meyer typedef enum { 828*a0483764SConrad Meyer 829*a0483764SConrad Meyer ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which 830*a0483764SConrad Meyer * the streaming API will refuse to allocate memory buffer 831*a0483764SConrad Meyer * in order to protect the host from unreasonable memory requirements. 832*a0483764SConrad Meyer * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. 833*a0483764SConrad Meyer * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */ 834*a0483764SConrad Meyer 835*a0483764SConrad Meyer /* note : additional experimental parameters are also available 836*a0483764SConrad Meyer * within the experimental section of the API. 837*a0483764SConrad Meyer * At the time of this writing, they include : 838*a0483764SConrad Meyer * ZSTD_c_format 839*a0483764SConrad Meyer * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. 840*a0483764SConrad Meyer * note : never ever use experimentalParam? names directly 841*a0483764SConrad Meyer */ 842*a0483764SConrad Meyer ZSTD_d_experimentalParam1=1000 843*a0483764SConrad Meyer 844*a0483764SConrad Meyer } ZSTD_dParameter; 845*a0483764SConrad Meyer 846*a0483764SConrad Meyer 847*a0483764SConrad Meyer /*! ZSTD_dParam_getBounds() : 848*a0483764SConrad Meyer * All parameters must belong to an interval with lower and upper bounds, 849*a0483764SConrad Meyer * otherwise they will either trigger an error or be automatically clamped. 850*a0483764SConrad Meyer * @return : a structure, ZSTD_bounds, which contains 851*a0483764SConrad Meyer * - an error status field, which must be tested using ZSTD_isError() 852*a0483764SConrad Meyer * - both lower and upper bounds, inclusive 853*a0483764SConrad Meyer */ 854*a0483764SConrad Meyer ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); 855*a0483764SConrad Meyer 856*a0483764SConrad Meyer /*! ZSTD_DCtx_setParameter() : 857*a0483764SConrad Meyer * Set one compression parameter, selected by enum ZSTD_dParameter. 858*a0483764SConrad Meyer * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). 859*a0483764SConrad Meyer * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). 860*a0483764SConrad Meyer * Setting a parameter is only possible during frame initialization (before starting decompression). 861*a0483764SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()). 862*a0483764SConrad Meyer */ 863*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); 864*a0483764SConrad Meyer 865*a0483764SConrad Meyer 866*a0483764SConrad Meyer /*! ZSTD_DCtx_loadDictionary() : 867*a0483764SConrad Meyer * Create an internal DDict from dict buffer, 868*a0483764SConrad Meyer * to be used to decompress next frames. 869*a0483764SConrad Meyer * The dictionary remains valid for all future frames, until explicitly invalidated. 870*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 871*a0483764SConrad Meyer * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, 872*a0483764SConrad Meyer * meaning "return to no-dictionary mode". 873*a0483764SConrad Meyer * Note 1 : Loading a dictionary involves building tables, 874*a0483764SConrad Meyer * which has a non-negligible impact on CPU usage and latency. 875*a0483764SConrad Meyer * It's recommended to "load once, use many times", to amortize the cost 876*a0483764SConrad Meyer * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. 877*a0483764SConrad Meyer * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. 878*a0483764SConrad Meyer * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of 879*a0483764SConrad Meyer * how dictionary content is loaded and interpreted. 880*a0483764SConrad Meyer */ 881*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); 882*a0483764SConrad Meyer 883*a0483764SConrad Meyer /*! ZSTD_DCtx_refDDict() : 884*a0483764SConrad Meyer * Reference a prepared dictionary, to be used to decompress next frames. 885*a0483764SConrad Meyer * The dictionary remains active for decompression of future frames using same DCtx. 886*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 887*a0483764SConrad Meyer * Note 1 : Currently, only one dictionary can be managed. 888*a0483764SConrad Meyer * Referencing a new dictionary effectively "discards" any previous one. 889*a0483764SConrad Meyer * Special: referencing a NULL DDict means "return to no-dictionary mode". 890*a0483764SConrad Meyer * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. 891*a0483764SConrad Meyer */ 892*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 893*a0483764SConrad Meyer 894*a0483764SConrad Meyer /*! ZSTD_DCtx_refPrefix() : 895*a0483764SConrad Meyer * Reference a prefix (single-usage dictionary) to decompress next frame. 896*a0483764SConrad Meyer * This is the reverse operation of ZSTD_CCtx_refPrefix(), 897*a0483764SConrad Meyer * and must use the same prefix as the one used during compression. 898*a0483764SConrad Meyer * Prefix is **only used once**. Reference is discarded at end of frame. 899*a0483764SConrad Meyer * End of frame is reached when ZSTD_decompressStream() returns 0. 900*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 901*a0483764SConrad Meyer * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary 902*a0483764SConrad Meyer * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. 903*a0483764SConrad Meyer * Prefix buffer must remain unmodified up to the end of frame, 904*a0483764SConrad Meyer * reached when ZSTD_decompressStream() returns 0. 905*a0483764SConrad Meyer * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). 906*a0483764SConrad Meyer * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) 907*a0483764SConrad Meyer * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. 908*a0483764SConrad Meyer * A full dictionary is more costly, as it requires building tables. 909*a0483764SConrad Meyer */ 910*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, 911*a0483764SConrad Meyer const void* prefix, size_t prefixSize); 912*a0483764SConrad Meyer 913*a0483764SConrad Meyer /*! ZSTD_DCtx_reset() : 914*a0483764SConrad Meyer * Return a DCtx to clean state. 915*a0483764SConrad Meyer * Session and parameters can be reset jointly or separately. 916*a0483764SConrad Meyer * Parameters can only be reset when no active frame is being decompressed. 917*a0483764SConrad Meyer * @return : 0, or an error code, which can be tested with ZSTD_isError() 918*a0483764SConrad Meyer */ 919*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); 920*a0483764SConrad Meyer 921*a0483764SConrad Meyer 922*a0483764SConrad Meyer 923*a0483764SConrad Meyer /**************************************************************************************** 924*a0483764SConrad Meyer * experimental API (static linking only) 925*a0483764SConrad Meyer **************************************************************************************** 926*a0483764SConrad Meyer * The following symbols and constants 927*a0483764SConrad Meyer * are not planned to join "stable API" status in the near future. 928*a0483764SConrad Meyer * They can still change in future versions. 929*a0483764SConrad Meyer * Some of them are planned to remain in the static_only section indefinitely. 930*a0483764SConrad Meyer * Some of them might be removed in the future (especially when redundant with existing stable functions) 931*a0483764SConrad Meyer * ***************************************************************************************/ 932*a0483764SConrad Meyer 933*a0483764SConrad Meyer #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */ 934*a0483764SConrad Meyer #define ZSTD_FRAMEHEADERSIZE_MIN 6 935*a0483764SConrad Meyer #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ 936*a0483764SConrad Meyer #define ZSTD_SKIPPABLEHEADERSIZE 8 937*a0483764SConrad Meyer 938*a0483764SConrad Meyer /* compression parameter bounds */ 9390c16b537SWarner Losh #define ZSTD_WINDOWLOG_MAX_32 30 9400c16b537SWarner Losh #define ZSTD_WINDOWLOG_MAX_64 31 941*a0483764SConrad Meyer #define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) 9420c16b537SWarner Losh #define ZSTD_WINDOWLOG_MIN 10 94319fcbaf1SConrad Meyer #define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) 9440c16b537SWarner Losh #define ZSTD_HASHLOG_MIN 6 94519fcbaf1SConrad Meyer #define ZSTD_CHAINLOG_MAX_32 29 94619fcbaf1SConrad Meyer #define ZSTD_CHAINLOG_MAX_64 30 947*a0483764SConrad Meyer #define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) 9480c16b537SWarner Losh #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN 9490c16b537SWarner Losh #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) 9500c16b537SWarner Losh #define ZSTD_SEARCHLOG_MIN 1 951*a0483764SConrad Meyer #define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ 952*a0483764SConrad Meyer #define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ 9530f743729SConrad Meyer #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX 9540f743729SConrad Meyer #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ 955*a0483764SConrad Meyer #define ZSTD_STRATEGY_MIN ZSTD_fast 956*a0483764SConrad Meyer #define ZSTD_STRATEGY_MAX ZSTD_btultra2 957*a0483764SConrad Meyer 958*a0483764SConrad Meyer 959*a0483764SConrad Meyer #define ZSTD_OVERLAPLOG_MIN 0 960*a0483764SConrad Meyer #define ZSTD_OVERLAPLOG_MAX 9 961*a0483764SConrad Meyer 962*a0483764SConrad Meyer #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame 963*a0483764SConrad Meyer * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size, 964*a0483764SConrad Meyer * to preserve host's memory from unreasonable requirements. 965*a0483764SConrad Meyer * This limit can be overriden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,). 966*a0483764SConrad Meyer * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */ 967*a0483764SConrad Meyer 968*a0483764SConrad Meyer 969*a0483764SConrad Meyer /* LDM parameter bounds */ 970*a0483764SConrad Meyer #define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN 971*a0483764SConrad Meyer #define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX 9720f743729SConrad Meyer #define ZSTD_LDM_MINMATCH_MIN 4 973*a0483764SConrad Meyer #define ZSTD_LDM_MINMATCH_MAX 4096 974*a0483764SConrad Meyer #define ZSTD_LDM_BUCKETSIZELOG_MIN 1 9750c16b537SWarner Losh #define ZSTD_LDM_BUCKETSIZELOG_MAX 8 976*a0483764SConrad Meyer #define ZSTD_LDM_HASHRATELOG_MIN 0 977*a0483764SConrad Meyer #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) 9780c16b537SWarner Losh 979*a0483764SConrad Meyer /* internal */ 980*a0483764SConrad Meyer #define ZSTD_HASHLOG3_MAX 17 9810c16b537SWarner Losh 9820f743729SConrad Meyer 9830c16b537SWarner Losh /* --- Advanced types --- */ 984*a0483764SConrad Meyer 985*a0483764SConrad Meyer typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; 9860c16b537SWarner Losh 9870c16b537SWarner Losh typedef struct { 9880c16b537SWarner Losh unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ 9890c16b537SWarner Losh unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ 9900c16b537SWarner Losh unsigned hashLog; /**< dispatch table : larger == faster, more memory */ 9910c16b537SWarner Losh unsigned searchLog; /**< nb of searches : larger == more compression, slower */ 992*a0483764SConrad Meyer unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ 9930c16b537SWarner Losh unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ 994*a0483764SConrad Meyer ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ 9950c16b537SWarner Losh } ZSTD_compressionParameters; 9960c16b537SWarner Losh 9970c16b537SWarner Losh typedef struct { 998*a0483764SConrad Meyer int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ 999*a0483764SConrad Meyer int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ 1000*a0483764SConrad Meyer int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ 10010c16b537SWarner Losh } ZSTD_frameParameters; 10020c16b537SWarner Losh 10030c16b537SWarner Losh typedef struct { 10040c16b537SWarner Losh ZSTD_compressionParameters cParams; 10050c16b537SWarner Losh ZSTD_frameParameters fParams; 10060c16b537SWarner Losh } ZSTD_parameters; 10070c16b537SWarner Losh 100819fcbaf1SConrad Meyer typedef enum { 100919fcbaf1SConrad Meyer ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ 1010*a0483764SConrad Meyer ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ 1011*a0483764SConrad Meyer ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ 101219fcbaf1SConrad Meyer } ZSTD_dictContentType_e; 101319fcbaf1SConrad Meyer 101419fcbaf1SConrad Meyer typedef enum { 101519fcbaf1SConrad Meyer ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ 1016*a0483764SConrad Meyer ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ 101719fcbaf1SConrad Meyer } ZSTD_dictLoadMethod_e; 101819fcbaf1SConrad Meyer 1019*a0483764SConrad Meyer typedef enum { 1020*a0483764SConrad Meyer /* Opened question : should we have a format ZSTD_f_auto ? 1021*a0483764SConrad Meyer * Today, it would mean exactly the same as ZSTD_f_zstd1. 1022*a0483764SConrad Meyer * But, in the future, should several formats become supported, 1023*a0483764SConrad Meyer * on the compression side, it would mean "default format". 1024*a0483764SConrad Meyer * On the decompression side, it would mean "automatic format detection", 1025*a0483764SConrad Meyer * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". 1026*a0483764SConrad Meyer * Since meaning is a little different, another option could be to define different enums for compression and decompression. 1027*a0483764SConrad Meyer * This question could be kept for later, when there are actually multiple formats to support, 1028*a0483764SConrad Meyer * but there is also the question of pinning enum values, and pinning value `0` is especially important */ 1029*a0483764SConrad Meyer ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ 1030*a0483764SConrad Meyer ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. 1031*a0483764SConrad Meyer * Useful to save 4 bytes per generated frame. 1032*a0483764SConrad Meyer * Decoder cannot recognise automatically this format, requiring this instruction. */ 1033*a0483764SConrad Meyer } ZSTD_format_e; 1034*a0483764SConrad Meyer 1035*a0483764SConrad Meyer typedef enum { 1036*a0483764SConrad Meyer /* Note: this enum and the behavior it controls are effectively internal 1037*a0483764SConrad Meyer * implementation details of the compressor. They are expected to continue 1038*a0483764SConrad Meyer * to evolve and should be considered only in the context of extremely 1039*a0483764SConrad Meyer * advanced performance tuning. 1040*a0483764SConrad Meyer * 1041*a0483764SConrad Meyer * Zstd currently supports the use of a CDict in two ways: 1042*a0483764SConrad Meyer * 1043*a0483764SConrad Meyer * - The contents of the CDict can be copied into the working context. This 1044*a0483764SConrad Meyer * means that the compression can search both the dictionary and input 1045*a0483764SConrad Meyer * while operating on a single set of internal tables. This makes 1046*a0483764SConrad Meyer * the compression faster per-byte of input. However, the initial copy of 1047*a0483764SConrad Meyer * the CDict's tables incurs a fixed cost at the beginning of the 1048*a0483764SConrad Meyer * compression. For small compressions (< 8 KB), that copy can dominate 1049*a0483764SConrad Meyer * the cost of the compression. 1050*a0483764SConrad Meyer * 1051*a0483764SConrad Meyer * - The CDict's tables can be used in-place. In this model, compression is 1052*a0483764SConrad Meyer * slower per input byte, because the compressor has to search two sets of 1053*a0483764SConrad Meyer * tables. However, this model incurs no start-up cost (as long as the 1054*a0483764SConrad Meyer * working context's tables can be reused). For small inputs, this can be 1055*a0483764SConrad Meyer * faster than copying the CDict's tables. 1056*a0483764SConrad Meyer * 1057*a0483764SConrad Meyer * Zstd has a simple internal heuristic that selects which strategy to use 1058*a0483764SConrad Meyer * at the beginning of a compression. However, if experimentation shows that 1059*a0483764SConrad Meyer * Zstd is making poor choices, it is possible to override that choice with 1060*a0483764SConrad Meyer * this enum. 1061*a0483764SConrad Meyer */ 1062*a0483764SConrad Meyer ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ 1063*a0483764SConrad Meyer ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ 1064*a0483764SConrad Meyer ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ 1065*a0483764SConrad Meyer } ZSTD_dictAttachPref_e; 10660c16b537SWarner Losh 10670c16b537SWarner Losh 10680c16b537SWarner Losh /*************************************** 10690c16b537SWarner Losh * Frame size functions 10700c16b537SWarner Losh ***************************************/ 10710c16b537SWarner Losh 10720c16b537SWarner Losh /*! ZSTD_findDecompressedSize() : 10730c16b537SWarner Losh * `src` should point the start of a series of ZSTD encoded and/or skippable frames 10740c16b537SWarner Losh * `srcSize` must be the _exact_ size of this series 10750c16b537SWarner Losh * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`) 10760c16b537SWarner Losh * @return : - decompressed size of all data in all successive frames 10770c16b537SWarner Losh * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN 10780c16b537SWarner Losh * - if an error occurred: ZSTD_CONTENTSIZE_ERROR 10790c16b537SWarner Losh * 10800c16b537SWarner Losh * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. 10810c16b537SWarner Losh * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. 10820c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 10830c16b537SWarner Losh * note 2 : decompressed size is always present when compression is done with ZSTD_compress() 10840c16b537SWarner Losh * note 3 : decompressed size can be very large (64-bits value), 10850c16b537SWarner Losh * potentially larger than what local system can handle as a single memory segment. 10860c16b537SWarner Losh * In which case, it's necessary to use streaming mode to decompress data. 10870c16b537SWarner Losh * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. 10880c16b537SWarner Losh * Always ensure result fits within application's authorized limits. 10890c16b537SWarner Losh * Each application can set its own limits. 10900c16b537SWarner Losh * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to 10910c16b537SWarner Losh * read each contained frame header. This is fast as most of the data is skipped, 10920c16b537SWarner Losh * however it does mean that all frame data must be present and valid. */ 10930c16b537SWarner Losh ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); 10940c16b537SWarner Losh 10950c16b537SWarner Losh /*! ZSTD_frameHeaderSize() : 1096*a0483764SConrad Meyer * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. 10970f743729SConrad Meyer * @return : size of the Frame Header, 10980f743729SConrad Meyer * or an error code (if srcSize is too small) */ 10990c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); 11000c16b537SWarner Losh 11010c16b537SWarner Losh 11020c16b537SWarner Losh /*************************************** 110319fcbaf1SConrad Meyer * Memory management 11040c16b537SWarner Losh ***************************************/ 11050c16b537SWarner Losh 11060c16b537SWarner Losh /*! ZSTD_estimate*() : 11070c16b537SWarner Losh * These functions make it possible to estimate memory usage 11080c16b537SWarner Losh * of a future {D,C}Ctx, before its creation. 11090c16b537SWarner Losh * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. 11100c16b537SWarner Losh * It will also consider src size to be arbitrarily "large", which is worst case. 11110c16b537SWarner Losh * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. 11120c16b537SWarner Losh * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. 1113*a0483764SConrad Meyer * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. 111419fcbaf1SConrad Meyer * Note : CCtx size estimation is only correct for single-threaded compression. */ 11150c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); 11160c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); 11170c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); 11180c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); 11190c16b537SWarner Losh 11200c16b537SWarner Losh /*! ZSTD_estimateCStreamSize() : 11210c16b537SWarner Losh * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. 11220c16b537SWarner Losh * It will also consider src size to be arbitrarily "large", which is worst case. 11230c16b537SWarner Losh * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. 11240c16b537SWarner Losh * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. 1125*a0483764SConrad Meyer * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. 112619fcbaf1SConrad Meyer * Note : CStream size estimation is only correct for single-threaded compression. 11270c16b537SWarner Losh * ZSTD_DStream memory budget depends on window Size. 11280c16b537SWarner Losh * This information can be passed manually, using ZSTD_estimateDStreamSize, 11290c16b537SWarner Losh * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); 11300c16b537SWarner Losh * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), 11310c16b537SWarner Losh * an internal ?Dict will be created, which additional size is not estimated here. 11320c16b537SWarner Losh * In this case, get total size by adding ZSTD_estimate?DictSize */ 11330c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); 11340c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); 11350c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); 11360c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); 11370c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); 11380c16b537SWarner Losh 11390c16b537SWarner Losh /*! ZSTD_estimate?DictSize() : 11400c16b537SWarner Losh * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). 114119fcbaf1SConrad Meyer * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). 114219fcbaf1SConrad Meyer * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. 11430c16b537SWarner Losh */ 11440c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); 11450c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); 11460c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); 11470c16b537SWarner Losh 114819fcbaf1SConrad Meyer /*! ZSTD_initStatic*() : 114919fcbaf1SConrad Meyer * Initialize an object using a pre-allocated fixed-size buffer. 115019fcbaf1SConrad Meyer * workspace: The memory area to emplace the object into. 115119fcbaf1SConrad Meyer * Provided pointer *must be 8-bytes aligned*. 115219fcbaf1SConrad Meyer * Buffer must outlive object. 115319fcbaf1SConrad Meyer * workspaceSize: Use ZSTD_estimate*Size() to determine 115419fcbaf1SConrad Meyer * how large workspace must be to support target scenario. 115519fcbaf1SConrad Meyer * @return : pointer to object (same address as workspace, just different type), 115619fcbaf1SConrad Meyer * or NULL if error (size too small, incorrect alignment, etc.) 115719fcbaf1SConrad Meyer * Note : zstd will never resize nor malloc() when using a static buffer. 115819fcbaf1SConrad Meyer * If the object requires more memory than available, 115919fcbaf1SConrad Meyer * zstd will just error out (typically ZSTD_error_memory_allocation). 116019fcbaf1SConrad Meyer * Note 2 : there is no corresponding "free" function. 116119fcbaf1SConrad Meyer * Since workspace is allocated externally, it must be freed externally too. 116219fcbaf1SConrad Meyer * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level 116319fcbaf1SConrad Meyer * into its associated cParams. 116419fcbaf1SConrad Meyer * Limitation 1 : currently not compatible with internal dictionary creation, triggered by 116519fcbaf1SConrad Meyer * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). 116619fcbaf1SConrad Meyer * Limitation 2 : static cctx currently not compatible with multi-threading. 116719fcbaf1SConrad Meyer * Limitation 3 : static dctx is incompatible with legacy support. 116819fcbaf1SConrad Meyer */ 116919fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); 117019fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ 117119fcbaf1SConrad Meyer 117219fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); 117319fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ 117419fcbaf1SConrad Meyer 117519fcbaf1SConrad Meyer ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( 117619fcbaf1SConrad Meyer void* workspace, size_t workspaceSize, 117719fcbaf1SConrad Meyer const void* dict, size_t dictSize, 117819fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 117919fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType, 118019fcbaf1SConrad Meyer ZSTD_compressionParameters cParams); 118119fcbaf1SConrad Meyer 118219fcbaf1SConrad Meyer ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( 118319fcbaf1SConrad Meyer void* workspace, size_t workspaceSize, 118419fcbaf1SConrad Meyer const void* dict, size_t dictSize, 118519fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 118619fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType); 118719fcbaf1SConrad Meyer 1188*a0483764SConrad Meyer 118919fcbaf1SConrad Meyer /*! Custom memory allocation : 119019fcbaf1SConrad Meyer * These prototypes make it possible to pass your own allocation/free functions. 119119fcbaf1SConrad Meyer * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. 119219fcbaf1SConrad Meyer * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. 119319fcbaf1SConrad Meyer */ 119419fcbaf1SConrad Meyer typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); 119519fcbaf1SConrad Meyer typedef void (*ZSTD_freeFunction) (void* opaque, void* address); 119619fcbaf1SConrad Meyer typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; 119719fcbaf1SConrad Meyer static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ 119819fcbaf1SConrad Meyer 119919fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); 120019fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); 120119fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); 120219fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); 120319fcbaf1SConrad Meyer 120419fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, 120519fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 120619fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType, 120719fcbaf1SConrad Meyer ZSTD_compressionParameters cParams, 120819fcbaf1SConrad Meyer ZSTD_customMem customMem); 120919fcbaf1SConrad Meyer 121019fcbaf1SConrad Meyer ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, 121119fcbaf1SConrad Meyer ZSTD_dictLoadMethod_e dictLoadMethod, 121219fcbaf1SConrad Meyer ZSTD_dictContentType_e dictContentType, 121319fcbaf1SConrad Meyer ZSTD_customMem customMem); 121419fcbaf1SConrad Meyer 121519fcbaf1SConrad Meyer 12160c16b537SWarner Losh 12170c16b537SWarner Losh /*************************************** 12180c16b537SWarner Losh * Advanced compression functions 12190c16b537SWarner Losh ***************************************/ 12200c16b537SWarner Losh 12210c16b537SWarner Losh /*! ZSTD_createCDict_byReference() : 12220c16b537SWarner Losh * Create a digested dictionary for compression 1223*a0483764SConrad Meyer * Dictionary content is just referenced, not duplicated. 1224*a0483764SConrad Meyer * As a consequence, `dictBuffer` **must** outlive CDict, 1225*a0483764SConrad Meyer * and its content must remain unmodified throughout the lifetime of CDict. */ 12260c16b537SWarner Losh ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); 12270c16b537SWarner Losh 12280c16b537SWarner Losh /*! ZSTD_getCParams() : 12290c16b537SWarner Losh * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. 12300c16b537SWarner Losh * `estimatedSrcSize` value is optional, select 0 if not known */ 12310c16b537SWarner Losh ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); 12320c16b537SWarner Losh 12330c16b537SWarner Losh /*! ZSTD_getParams() : 12340c16b537SWarner Losh * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. 1235052d3c12SConrad Meyer * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ 12360c16b537SWarner Losh ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); 12370c16b537SWarner Losh 12380c16b537SWarner Losh /*! ZSTD_checkCParams() : 12390c16b537SWarner Losh * Ensure param values remain within authorized range */ 12400c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); 12410c16b537SWarner Losh 12420c16b537SWarner Losh /*! ZSTD_adjustCParams() : 12430c16b537SWarner Losh * optimize params for a given `srcSize` and `dictSize`. 12440c16b537SWarner Losh * both values are optional, select `0` if unknown. */ 12450c16b537SWarner Losh ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); 12460c16b537SWarner Losh 12470c16b537SWarner Losh /*! ZSTD_compress_advanced() : 1248*a0483764SConrad Meyer * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */ 12490c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, 12500c16b537SWarner Losh void* dst, size_t dstCapacity, 12510c16b537SWarner Losh const void* src, size_t srcSize, 12520c16b537SWarner Losh const void* dict,size_t dictSize, 12530c16b537SWarner Losh ZSTD_parameters params); 12540c16b537SWarner Losh 12550c16b537SWarner Losh /*! ZSTD_compress_usingCDict_advanced() : 12560c16b537SWarner Losh * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ 12570c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, 12580c16b537SWarner Losh void* dst, size_t dstCapacity, 12590c16b537SWarner Losh const void* src, size_t srcSize, 1260*a0483764SConrad Meyer const ZSTD_CDict* cdict, 1261*a0483764SConrad Meyer ZSTD_frameParameters fParams); 12620c16b537SWarner Losh 12630c16b537SWarner Losh 1264*a0483764SConrad Meyer /*! ZSTD_CCtx_loadDictionary_byReference() : 1265*a0483764SConrad Meyer * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. 1266*a0483764SConrad Meyer * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ 1267*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); 1268*a0483764SConrad Meyer 1269*a0483764SConrad Meyer /*! ZSTD_CCtx_loadDictionary_advanced() : 1270*a0483764SConrad Meyer * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over 1271*a0483764SConrad Meyer * how to load the dictionary (by copy ? by reference ?) 1272*a0483764SConrad Meyer * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ 1273*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); 1274*a0483764SConrad Meyer 1275*a0483764SConrad Meyer /*! ZSTD_CCtx_refPrefix_advanced() : 1276*a0483764SConrad Meyer * Same as ZSTD_CCtx_refPrefix(), but gives finer control over 1277*a0483764SConrad Meyer * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ 1278*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); 1279*a0483764SConrad Meyer 1280*a0483764SConrad Meyer /* === experimental parameters === */ 1281*a0483764SConrad Meyer /* these parameters can be used with ZSTD_setParameter() 1282*a0483764SConrad Meyer * they are not guaranteed to remain supported in the future */ 1283*a0483764SConrad Meyer 1284*a0483764SConrad Meyer /* Enables rsyncable mode, 1285*a0483764SConrad Meyer * which makes compressed files more rsync friendly 1286*a0483764SConrad Meyer * by adding periodic synchronization points to the compressed data. 1287*a0483764SConrad Meyer * The target average block size is ZSTD_c_jobSize / 2. 1288*a0483764SConrad Meyer * It's possible to modify the job size to increase or decrease 1289*a0483764SConrad Meyer * the granularity of the synchronization point. 1290*a0483764SConrad Meyer * Once the jobSize is smaller than the window size, 1291*a0483764SConrad Meyer * it will result in compression ratio degradation. 1292*a0483764SConrad Meyer * NOTE 1: rsyncable mode only works when multithreading is enabled. 1293*a0483764SConrad Meyer * NOTE 2: rsyncable performs poorly in combination with long range mode, 1294*a0483764SConrad Meyer * since it will decrease the effectiveness of synchronization points, 1295*a0483764SConrad Meyer * though mileage may vary. 1296*a0483764SConrad Meyer * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. 1297*a0483764SConrad Meyer * If the selected compression level is already running significantly slower, 1298*a0483764SConrad Meyer * the overall speed won't be significantly impacted. 1299*a0483764SConrad Meyer */ 1300*a0483764SConrad Meyer #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 1301*a0483764SConrad Meyer 1302*a0483764SConrad Meyer /* Select a compression format. 1303*a0483764SConrad Meyer * The value must be of type ZSTD_format_e. 1304*a0483764SConrad Meyer * See ZSTD_format_e enum definition for details */ 1305*a0483764SConrad Meyer #define ZSTD_c_format ZSTD_c_experimentalParam2 1306*a0483764SConrad Meyer 1307*a0483764SConrad Meyer /* Force back-reference distances to remain < windowSize, 1308*a0483764SConrad Meyer * even when referencing into Dictionary content (default:0) */ 1309*a0483764SConrad Meyer #define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 1310*a0483764SConrad Meyer 1311*a0483764SConrad Meyer /* Controls whether the contents of a CDict 1312*a0483764SConrad Meyer * are used in place, or copied into the working context. 1313*a0483764SConrad Meyer * Accepts values from the ZSTD_dictAttachPref_e enum. 1314*a0483764SConrad Meyer * See the comments on that enum for an explanation of the feature. */ 1315*a0483764SConrad Meyer #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 1316*a0483764SConrad Meyer 1317*a0483764SConrad Meyer /*! ZSTD_CCtx_getParameter() : 1318*a0483764SConrad Meyer * Get the requested compression parameter value, selected by enum ZSTD_cParameter, 1319*a0483764SConrad Meyer * and store it into int* value. 1320*a0483764SConrad Meyer * @return : 0, or an error code (which can be tested with ZSTD_isError()). 1321*a0483764SConrad Meyer */ 1322*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); 1323*a0483764SConrad Meyer 1324*a0483764SConrad Meyer 1325*a0483764SConrad Meyer /*! ZSTD_CCtx_params : 1326*a0483764SConrad Meyer * Quick howto : 1327*a0483764SConrad Meyer * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure 1328*a0483764SConrad Meyer * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into 1329*a0483764SConrad Meyer * an existing ZSTD_CCtx_params structure. 1330*a0483764SConrad Meyer * This is similar to 1331*a0483764SConrad Meyer * ZSTD_CCtx_setParameter(). 1332*a0483764SConrad Meyer * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to 1333*a0483764SConrad Meyer * an existing CCtx. 1334*a0483764SConrad Meyer * These parameters will be applied to 1335*a0483764SConrad Meyer * all subsequent frames. 1336*a0483764SConrad Meyer * - ZSTD_compressStream2() : Do compression using the CCtx. 1337*a0483764SConrad Meyer * - ZSTD_freeCCtxParams() : Free the memory. 1338*a0483764SConrad Meyer * 1339*a0483764SConrad Meyer * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() 1340*a0483764SConrad Meyer * for static allocation of CCtx for single-threaded compression. 1341*a0483764SConrad Meyer */ 1342*a0483764SConrad Meyer ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); 1343*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); 1344*a0483764SConrad Meyer 1345*a0483764SConrad Meyer /*! ZSTD_CCtxParams_reset() : 1346*a0483764SConrad Meyer * Reset params to default values. 1347*a0483764SConrad Meyer */ 1348*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); 1349*a0483764SConrad Meyer 1350*a0483764SConrad Meyer /*! ZSTD_CCtxParams_init() : 1351*a0483764SConrad Meyer * Initializes the compression parameters of cctxParams according to 1352*a0483764SConrad Meyer * compression level. All other parameters are reset to their default values. 1353*a0483764SConrad Meyer */ 1354*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); 1355*a0483764SConrad Meyer 1356*a0483764SConrad Meyer /*! ZSTD_CCtxParams_init_advanced() : 1357*a0483764SConrad Meyer * Initializes the compression and frame parameters of cctxParams according to 1358*a0483764SConrad Meyer * params. All other parameters are reset to their default values. 1359*a0483764SConrad Meyer */ 1360*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); 1361*a0483764SConrad Meyer 1362*a0483764SConrad Meyer /*! ZSTD_CCtxParam_setParameter() : 1363*a0483764SConrad Meyer * Similar to ZSTD_CCtx_setParameter. 1364*a0483764SConrad Meyer * Set one compression parameter, selected by enum ZSTD_cParameter. 1365*a0483764SConrad Meyer * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). 1366*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 1367*a0483764SConrad Meyer */ 1368*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); 1369*a0483764SConrad Meyer 1370*a0483764SConrad Meyer /*! ZSTD_CCtxParam_getParameter() : 1371*a0483764SConrad Meyer * Similar to ZSTD_CCtx_getParameter. 1372*a0483764SConrad Meyer * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. 1373*a0483764SConrad Meyer * @result : 0, or an error code (which can be tested with ZSTD_isError()). 1374*a0483764SConrad Meyer */ 1375*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); 1376*a0483764SConrad Meyer 1377*a0483764SConrad Meyer /*! ZSTD_CCtx_setParametersUsingCCtxParams() : 1378*a0483764SConrad Meyer * Apply a set of ZSTD_CCtx_params to the compression context. 1379*a0483764SConrad Meyer * This can be done even after compression is started, 1380*a0483764SConrad Meyer * if nbWorkers==0, this will have no impact until a new compression is started. 1381*a0483764SConrad Meyer * if nbWorkers>=1, new parameters will be picked up at next job, 1382*a0483764SConrad Meyer * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). 1383*a0483764SConrad Meyer */ 1384*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( 1385*a0483764SConrad Meyer ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); 1386*a0483764SConrad Meyer 1387*a0483764SConrad Meyer /*! ZSTD_compressStream2_simpleArgs() : 1388*a0483764SConrad Meyer * Same as ZSTD_compressStream2(), 1389*a0483764SConrad Meyer * but using only integral types as arguments. 1390*a0483764SConrad Meyer * This variant might be helpful for binders from dynamic languages 1391*a0483764SConrad Meyer * which have troubles handling structures containing memory pointers. 1392*a0483764SConrad Meyer */ 1393*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( 1394*a0483764SConrad Meyer ZSTD_CCtx* cctx, 1395*a0483764SConrad Meyer void* dst, size_t dstCapacity, size_t* dstPos, 1396*a0483764SConrad Meyer const void* src, size_t srcSize, size_t* srcPos, 1397*a0483764SConrad Meyer ZSTD_EndDirective endOp); 1398*a0483764SConrad Meyer 1399*a0483764SConrad Meyer 1400*a0483764SConrad Meyer /*************************************** 1401*a0483764SConrad Meyer * Advanced decompression functions 1402*a0483764SConrad Meyer ***************************************/ 14030c16b537SWarner Losh 14040c16b537SWarner Losh /*! ZSTD_isFrame() : 14050c16b537SWarner Losh * Tells if the content of `buffer` starts with a valid Frame Identifier. 14060c16b537SWarner Losh * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. 14070c16b537SWarner Losh * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. 14080c16b537SWarner Losh * Note 3 : Skippable Frame Identifiers are considered valid. */ 14090c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); 14100c16b537SWarner Losh 14110c16b537SWarner Losh /*! ZSTD_createDDict_byReference() : 14120c16b537SWarner Losh * Create a digested dictionary, ready to start decompression operation without startup delay. 14130c16b537SWarner Losh * Dictionary content is referenced, and therefore stays in dictBuffer. 14140c16b537SWarner Losh * It is important that dictBuffer outlives DDict, 14150c16b537SWarner Losh * it must remain read accessible throughout the lifetime of DDict */ 14160c16b537SWarner Losh ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); 14170c16b537SWarner Losh 14180c16b537SWarner Losh 14190c16b537SWarner Losh /*! ZSTD_getDictID_fromDict() : 14200c16b537SWarner Losh * Provides the dictID stored within dictionary. 14210c16b537SWarner Losh * if @return == 0, the dictionary is not conformant with Zstandard specification. 14220c16b537SWarner Losh * It can still be loaded, but as a content-only dictionary. */ 14230c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); 14240c16b537SWarner Losh 14250c16b537SWarner Losh /*! ZSTD_getDictID_fromDDict() : 14260c16b537SWarner Losh * Provides the dictID of the dictionary loaded into `ddict`. 14270c16b537SWarner Losh * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. 14280c16b537SWarner Losh * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ 14290c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); 14300c16b537SWarner Losh 14310c16b537SWarner Losh /*! ZSTD_getDictID_fromFrame() : 14320c16b537SWarner Losh * Provides the dictID required to decompressed the frame stored within `src`. 14330c16b537SWarner Losh * If @return == 0, the dictID could not be decoded. 14340c16b537SWarner Losh * This could for one of the following reasons : 14350c16b537SWarner Losh * - The frame does not require a dictionary to be decoded (most common case). 14360c16b537SWarner Losh * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. 14370c16b537SWarner Losh * Note : this use case also happens when using a non-conformant dictionary. 14380c16b537SWarner Losh * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). 14390c16b537SWarner Losh * - This is not a Zstandard frame. 14400c16b537SWarner Losh * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ 14410c16b537SWarner Losh ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); 14420c16b537SWarner Losh 1443*a0483764SConrad Meyer /*! ZSTD_DCtx_loadDictionary_byReference() : 1444*a0483764SConrad Meyer * Same as ZSTD_DCtx_loadDictionary(), 1445*a0483764SConrad Meyer * but references `dict` content instead of copying it into `dctx`. 1446*a0483764SConrad Meyer * This saves memory if `dict` remains around., 1447*a0483764SConrad Meyer * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ 1448*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); 1449*a0483764SConrad Meyer 1450*a0483764SConrad Meyer /*! ZSTD_DCtx_loadDictionary_advanced() : 1451*a0483764SConrad Meyer * Same as ZSTD_DCtx_loadDictionary(), 1452*a0483764SConrad Meyer * but gives direct control over 1453*a0483764SConrad Meyer * how to load the dictionary (by copy ? by reference ?) 1454*a0483764SConrad Meyer * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ 1455*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); 1456*a0483764SConrad Meyer 1457*a0483764SConrad Meyer /*! ZSTD_DCtx_refPrefix_advanced() : 1458*a0483764SConrad Meyer * Same as ZSTD_DCtx_refPrefix(), but gives finer control over 1459*a0483764SConrad Meyer * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ 1460*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); 1461*a0483764SConrad Meyer 1462*a0483764SConrad Meyer /*! ZSTD_DCtx_setMaxWindowSize() : 1463*a0483764SConrad Meyer * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. 1464*a0483764SConrad Meyer * This protects a decoder context from reserving too much memory for itself (potential attack scenario). 1465*a0483764SConrad Meyer * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. 1466*a0483764SConrad Meyer * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) 1467*a0483764SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()). 1468*a0483764SConrad Meyer */ 1469*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); 1470*a0483764SConrad Meyer 1471*a0483764SConrad Meyer /* ZSTD_d_format 1472*a0483764SConrad Meyer * experimental parameter, 1473*a0483764SConrad Meyer * allowing selection between ZSTD_format_e input compression formats 1474*a0483764SConrad Meyer */ 1475*a0483764SConrad Meyer #define ZSTD_d_format ZSTD_d_experimentalParam1 1476*a0483764SConrad Meyer 1477*a0483764SConrad Meyer /*! ZSTD_DCtx_setFormat() : 1478*a0483764SConrad Meyer * Instruct the decoder context about what kind of data to decode next. 1479*a0483764SConrad Meyer * This instruction is mandatory to decode data without a fully-formed header, 1480*a0483764SConrad Meyer * such ZSTD_f_zstd1_magicless for example. 1481*a0483764SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ 1482*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); 1483*a0483764SConrad Meyer 1484*a0483764SConrad Meyer /*! ZSTD_decompressStream_simpleArgs() : 1485*a0483764SConrad Meyer * Same as ZSTD_decompressStream(), 1486*a0483764SConrad Meyer * but using only integral types as arguments. 1487*a0483764SConrad Meyer * This can be helpful for binders from dynamic languages 1488*a0483764SConrad Meyer * which have troubles handling structures containing memory pointers. 1489*a0483764SConrad Meyer */ 1490*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( 1491*a0483764SConrad Meyer ZSTD_DCtx* dctx, 1492*a0483764SConrad Meyer void* dst, size_t dstCapacity, size_t* dstPos, 1493*a0483764SConrad Meyer const void* src, size_t srcSize, size_t* srcPos); 1494*a0483764SConrad Meyer 14950c16b537SWarner Losh 14960c16b537SWarner Losh /******************************************************************** 14970c16b537SWarner Losh * Advanced streaming functions 1498*a0483764SConrad Meyer * Warning : most of these functions are now redundant with the Advanced API. 1499*a0483764SConrad Meyer * Once Advanced API reaches "stable" status, 1500*a0483764SConrad Meyer * redundant functions will be deprecated, and then at some point removed. 15010c16b537SWarner Losh ********************************************************************/ 15020c16b537SWarner Losh 15030c16b537SWarner Losh /*===== Advanced Streaming compression functions =====*/ 1504052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */ 15050c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ 15060c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, 1507052d3c12SConrad Meyer ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ 15080c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ 1509052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ 15100c16b537SWarner Losh 15110c16b537SWarner Losh /*! ZSTD_resetCStream() : 1512*a0483764SConrad Meyer * start a new frame, using same parameters from previous frame. 15130f743729SConrad Meyer * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. 15140c16b537SWarner Losh * Note that zcs must be init at least once before using ZSTD_resetCStream(). 1515052d3c12SConrad Meyer * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. 15160c16b537SWarner Losh * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. 1517052d3c12SConrad Meyer * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, 151819fcbaf1SConrad Meyer * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. 15190f743729SConrad Meyer * @return : 0, or an error code (which can be tested using ZSTD_isError()) 15200f743729SConrad Meyer */ 15210c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); 15220c16b537SWarner Losh 15230c16b537SWarner Losh 152419fcbaf1SConrad Meyer typedef struct { 15250f743729SConrad Meyer unsigned long long ingested; /* nb input bytes read and buffered */ 15260f743729SConrad Meyer unsigned long long consumed; /* nb input bytes actually compressed */ 15270f743729SConrad Meyer unsigned long long produced; /* nb of compressed bytes generated and buffered */ 15280f743729SConrad Meyer unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ 15290f743729SConrad Meyer unsigned currentJobID; /* MT only : latest started job nb */ 15300f743729SConrad Meyer unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ 153119fcbaf1SConrad Meyer } ZSTD_frameProgression; 153219fcbaf1SConrad Meyer 153319fcbaf1SConrad Meyer /* ZSTD_getFrameProgression() : 153419fcbaf1SConrad Meyer * tells how much data has been ingested (read from input) 153519fcbaf1SConrad Meyer * consumed (input actually compressed) and produced (output) for current frame. 15360f743729SConrad Meyer * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. 15370f743729SConrad Meyer * Aggregates progression inside active worker threads. 153819fcbaf1SConrad Meyer */ 15390f743729SConrad Meyer ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); 15400f743729SConrad Meyer 15410f743729SConrad Meyer /*! ZSTD_toFlushNow() : 15420f743729SConrad Meyer * Tell how many bytes are ready to be flushed immediately. 15430f743729SConrad Meyer * Useful for multithreading scenarios (nbWorkers >= 1). 15440f743729SConrad Meyer * Probe the oldest active job, defined as oldest job not yet entirely flushed, 15450f743729SConrad Meyer * and check its output buffer. 15460f743729SConrad Meyer * @return : amount of data stored in oldest job and ready to be flushed immediately. 15470f743729SConrad Meyer * if @return == 0, it means either : 15480f743729SConrad Meyer * + there is no active job (could be checked with ZSTD_frameProgression()), or 15490f743729SConrad Meyer * + oldest job is still actively compressing data, 15500f743729SConrad Meyer * but everything it has produced has also been flushed so far, 1551*a0483764SConrad Meyer * therefore flush speed is limited by production speed of oldest job 1552*a0483764SConrad Meyer * irrespective of the speed of concurrent (and newer) jobs. 15530f743729SConrad Meyer */ 15540f743729SConrad Meyer ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); 155519fcbaf1SConrad Meyer 155619fcbaf1SConrad Meyer 15570c16b537SWarner Losh /*===== Advanced Streaming decompression functions =====*/ 15580c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */ 15590c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */ 15600c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ 15610c16b537SWarner Losh 15620c16b537SWarner Losh 15630c16b537SWarner Losh /********************************************************************* 15640c16b537SWarner Losh * Buffer-less and synchronous inner streaming functions 15650c16b537SWarner Losh * 15660c16b537SWarner Losh * This is an advanced API, giving full control over buffer management, for users which need direct control over memory. 15670c16b537SWarner Losh * But it's also a complex one, with several restrictions, documented below. 15680c16b537SWarner Losh * Prefer normal streaming API for an easier experience. 15690c16b537SWarner Losh ********************************************************************* */ 15700c16b537SWarner Losh 15710c16b537SWarner Losh /** 15720c16b537SWarner Losh Buffer-less streaming compression (synchronous mode) 15730c16b537SWarner Losh 15740c16b537SWarner Losh A ZSTD_CCtx object is required to track streaming operations. 15750c16b537SWarner Losh Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. 15760c16b537SWarner Losh ZSTD_CCtx object can be re-used multiple times within successive compression operations. 15770c16b537SWarner Losh 15780c16b537SWarner Losh Start by initializing a context. 15790c16b537SWarner Losh Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, 15800c16b537SWarner Losh or ZSTD_compressBegin_advanced(), for finer parameter control. 15810c16b537SWarner Losh It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() 15820c16b537SWarner Losh 15830c16b537SWarner Losh Then, consume your input using ZSTD_compressContinue(). 15840c16b537SWarner Losh There are some important considerations to keep in mind when using this advanced function : 15850c16b537SWarner Losh - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. 15860c16b537SWarner Losh - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. 15870c16b537SWarner Losh - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. 15880c16b537SWarner Losh Worst case evaluation is provided by ZSTD_compressBound(). 15890c16b537SWarner Losh ZSTD_compressContinue() doesn't guarantee recover after a failed compression. 15900c16b537SWarner Losh - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). 15910c16b537SWarner Losh It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) 15920c16b537SWarner Losh - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. 15930c16b537SWarner Losh In which case, it will "discard" the relevant memory section from its history. 15940c16b537SWarner Losh 15950c16b537SWarner Losh Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. 15960c16b537SWarner Losh It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. 15970c16b537SWarner Losh Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. 15980c16b537SWarner Losh 15990c16b537SWarner Losh `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. 16000c16b537SWarner Losh */ 16010c16b537SWarner Losh 16020c16b537SWarner Losh /*===== Buffer-less streaming compression functions =====*/ 16030c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); 16040c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); 1605052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ 16060c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ 1607052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ 1608052d3c12SConrad Meyer ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ 16090c16b537SWarner Losh 16100c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 16110c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 16120c16b537SWarner Losh 16130c16b537SWarner Losh 16140c16b537SWarner Losh /*- 16150c16b537SWarner Losh Buffer-less streaming decompression (synchronous mode) 16160c16b537SWarner Losh 16170c16b537SWarner Losh A ZSTD_DCtx object is required to track streaming operations. 16180c16b537SWarner Losh Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. 16190c16b537SWarner Losh A ZSTD_DCtx object can be re-used multiple times. 16200c16b537SWarner Losh 16210c16b537SWarner Losh First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). 16220c16b537SWarner Losh Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. 16230c16b537SWarner Losh Data fragment must be large enough to ensure successful decoding. 16240c16b537SWarner Losh `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. 16250c16b537SWarner Losh @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. 16260c16b537SWarner Losh >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. 16270c16b537SWarner Losh errorCode, which can be tested using ZSTD_isError(). 16280c16b537SWarner Losh 16290c16b537SWarner Losh It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, 16300c16b537SWarner Losh such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). 16310c16b537SWarner Losh Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. 16320c16b537SWarner Losh As a consequence, check that values remain within valid application range. 16330c16b537SWarner Losh For example, do not allocate memory blindly, check that `windowSize` is within expectation. 16340c16b537SWarner Losh Each application can set its own limits, depending on local restrictions. 16350c16b537SWarner Losh For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. 16360c16b537SWarner Losh 16370c16b537SWarner Losh ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. 16380c16b537SWarner Losh ZSTD_decompressContinue() is very sensitive to contiguity, 16390c16b537SWarner Losh if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, 16400c16b537SWarner Losh or that previous contiguous segment is large enough to properly handle maximum back-reference distance. 16410c16b537SWarner Losh There are multiple ways to guarantee this condition. 16420c16b537SWarner Losh 16430c16b537SWarner Losh The most memory efficient way is to use a round buffer of sufficient size. 16440c16b537SWarner Losh Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), 16450c16b537SWarner Losh which can @return an error code if required value is too large for current system (in 32-bits mode). 16460c16b537SWarner Losh In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, 16470c16b537SWarner Losh up to the moment there is not enough room left in the buffer to guarantee decoding another full block, 16480c16b537SWarner Losh which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. 16490c16b537SWarner Losh At which point, decoding can resume from the beginning of the buffer. 16500c16b537SWarner Losh Note that already decoded data stored in the buffer should be flushed before being overwritten. 16510c16b537SWarner Losh 16520c16b537SWarner Losh There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. 16530c16b537SWarner Losh 16540c16b537SWarner Losh Finally, if you control the compression process, you can also ignore all buffer size rules, 16550c16b537SWarner Losh as long as the encoder and decoder progress in "lock-step", 16560c16b537SWarner Losh aka use exactly the same buffer sizes, break contiguity at the same place, etc. 16570c16b537SWarner Losh 16580c16b537SWarner Losh Once buffers are setup, start decompression, with ZSTD_decompressBegin(). 16590c16b537SWarner Losh If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). 16600c16b537SWarner Losh 16610c16b537SWarner Losh Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. 16620c16b537SWarner Losh ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). 16630c16b537SWarner Losh ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. 16640c16b537SWarner Losh 16650c16b537SWarner Losh @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). 16660c16b537SWarner Losh It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. 16670c16b537SWarner Losh It can also be an error code, which can be tested with ZSTD_isError(). 16680c16b537SWarner Losh 16690c16b537SWarner Losh A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. 16700c16b537SWarner Losh Context can then be reset to start a new decompression. 16710c16b537SWarner Losh 16720c16b537SWarner Losh Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). 16730c16b537SWarner Losh This information is not required to properly decode a frame. 16740c16b537SWarner Losh 16750c16b537SWarner Losh == Special case : skippable frames == 16760c16b537SWarner Losh 16770c16b537SWarner Losh Skippable frames allow integration of user-defined data into a flow of concatenated frames. 16780c16b537SWarner Losh Skippable frames will be ignored (skipped) by decompressor. 16790c16b537SWarner Losh The format of skippable frames is as follows : 16800c16b537SWarner Losh a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F 16810c16b537SWarner Losh b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits 16820c16b537SWarner Losh c) Frame Content - any content (User Data) of length equal to Frame Size 16830c16b537SWarner Losh For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. 16840c16b537SWarner Losh For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. 16850c16b537SWarner Losh */ 16860c16b537SWarner Losh 16870c16b537SWarner Losh /*===== Buffer-less streaming decompression functions =====*/ 16880c16b537SWarner Losh typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; 16890c16b537SWarner Losh typedef struct { 16900c16b537SWarner Losh unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ 16910c16b537SWarner Losh unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ 16920c16b537SWarner Losh unsigned blockSizeMax; 16930c16b537SWarner Losh ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ 16940c16b537SWarner Losh unsigned headerSize; 16950c16b537SWarner Losh unsigned dictID; 16960c16b537SWarner Losh unsigned checksumFlag; 16970c16b537SWarner Losh } ZSTD_frameHeader; 1698*a0483764SConrad Meyer 16990f743729SConrad Meyer /** ZSTD_getFrameHeader() : 17000f743729SConrad Meyer * decode Frame Header, or requires larger `srcSize`. 17010f743729SConrad Meyer * @return : 0, `zfhPtr` is correctly filled, 17020f743729SConrad Meyer * >0, `srcSize` is too small, value is wanted `srcSize` amount, 17030f743729SConrad Meyer * or an error code, which can be tested using ZSTD_isError() */ 17040c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ 1705*a0483764SConrad Meyer /*! ZSTD_getFrameHeader_advanced() : 1706*a0483764SConrad Meyer * same as ZSTD_getFrameHeader(), 1707*a0483764SConrad Meyer * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ 1708*a0483764SConrad Meyer ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); 17090c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ 17100c16b537SWarner Losh 17110c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); 17120c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); 17130c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 17140c16b537SWarner Losh 17150c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); 17160c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 17170c16b537SWarner Losh 17180c16b537SWarner Losh /* misc */ 17190c16b537SWarner Losh ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); 17200c16b537SWarner Losh typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; 17210c16b537SWarner Losh ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); 17220c16b537SWarner Losh 17230c16b537SWarner Losh 17240c16b537SWarner Losh 17250c16b537SWarner Losh 17260c16b537SWarner Losh /* ============================ */ 17270c16b537SWarner Losh /** Block level API */ 17280c16b537SWarner Losh /* ============================ */ 17290c16b537SWarner Losh 17300c16b537SWarner Losh /*! 17310c16b537SWarner Losh Block functions produce and decode raw zstd blocks, without frame metadata. 17320c16b537SWarner Losh Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). 17330c16b537SWarner Losh User will have to take in charge required information to regenerate data, such as compressed and content sizes. 17340c16b537SWarner Losh 17350c16b537SWarner Losh A few rules to respect : 17360c16b537SWarner Losh - Compressing and decompressing require a context structure 17370c16b537SWarner Losh + Use ZSTD_createCCtx() and ZSTD_createDCtx() 17380c16b537SWarner Losh - It is necessary to init context before starting 17390c16b537SWarner Losh + compression : any ZSTD_compressBegin*() variant, including with dictionary 17400c16b537SWarner Losh + decompression : any ZSTD_decompressBegin*() variant, including with dictionary 17410c16b537SWarner Losh + copyCCtx() and copyDCtx() can be used too 17420c16b537SWarner Losh - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB 17430c16b537SWarner Losh + If input is larger than a block size, it's necessary to split input data into multiple blocks 1744*a0483764SConrad Meyer + For inputs larger than a single block, really consider using regular ZSTD_compress() instead. 17450c16b537SWarner Losh Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. 17460c16b537SWarner Losh - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. 1747*a0483764SConrad Meyer In which case, nothing is produced into `dst` ! 17480c16b537SWarner Losh + User must test for such outcome and deal directly with uncompressed data 17490c16b537SWarner Losh + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! 17500c16b537SWarner Losh + In case of multiple successive blocks, should some of them be uncompressed, 17510c16b537SWarner Losh decoder must be informed of their existence in order to follow proper history. 17520c16b537SWarner Losh Use ZSTD_insertBlock() for such a case. 17530c16b537SWarner Losh */ 17540c16b537SWarner Losh 17550c16b537SWarner Losh /*===== Raw zstd block functions =====*/ 17560c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); 17570c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 17580c16b537SWarner Losh ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 175919fcbaf1SConrad Meyer ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ 17600c16b537SWarner Losh 17610c16b537SWarner Losh 17620c16b537SWarner Losh #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ 17630c16b537SWarner Losh 17640c16b537SWarner Losh #if defined (__cplusplus) 17650c16b537SWarner Losh } 17660c16b537SWarner Losh #endif 1767