xref: /freebsd/sys/contrib/zstd/lib/compress/zstdmt_compress.h (revision 36712a94975f5bd0d26c85377283b49a2369c82f)
1 /*
2  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under both the BSD-style license (found in the
6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7  * in the COPYING file in the root directory of this source tree).
8  * You may select, at your option, one of the above-listed licenses.
9  */
10 
11  #ifndef ZSTDMT_COMPRESS_H
12  #define ZSTDMT_COMPRESS_H
13 
14  #if defined (__cplusplus)
15  extern "C" {
16  #endif
17 
18 
19 /* Note : This is an internal API.
20  *        These APIs used to be exposed with ZSTDLIB_API,
21  *        because it used to be the only way to invoke MT compression.
22  *        Now, it's recommended to use ZSTD_compress2 and ZSTD_compressStream2()
23  *        instead.
24  *
25  *        If you depend on these APIs and can't switch, then define
26  *        ZSTD_LEGACY_MULTITHREADED_API when making the dynamic library.
27  *        However, we may completely remove these functions in a future
28  *        release, so please switch soon.
29  *
30  *        This API requires ZSTD_MULTITHREAD to be defined during compilation,
31  *        otherwise ZSTDMT_createCCtx*() will fail.
32  */
33 
34 #ifdef ZSTD_LEGACY_MULTITHREADED_API
35 #  define ZSTDMT_API ZSTDLIB_API
36 #else
37 #  define ZSTDMT_API
38 #endif
39 
40 /* ===   Dependencies   === */
41 #include <stddef.h>                /* size_t */
42 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_parameters */
43 #include "zstd.h"            /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
44 
45 
46 /* ===   Constants   === */
47 #ifndef ZSTDMT_NBWORKERS_MAX
48 #  define ZSTDMT_NBWORKERS_MAX 200
49 #endif
50 #ifndef ZSTDMT_JOBSIZE_MIN
51 #  define ZSTDMT_JOBSIZE_MIN (1 MB)
52 #endif
53 #define ZSTDMT_JOBLOG_MAX   (MEM_32bits() ? 29 : 30)
54 #define ZSTDMT_JOBSIZE_MAX  (MEM_32bits() ? (512 MB) : (1024 MB))
55 
56 
57 /* ===   Memory management   === */
58 typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
59 /* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
60 ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
61 /* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
62 ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
63                                                     ZSTD_customMem cMem);
64 ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
65 
66 ZSTDMT_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
67 
68 
69 /* ===   Simple one-pass compression function   === */
70 
71 ZSTDMT_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
72                                        void* dst, size_t dstCapacity,
73                                  const void* src, size_t srcSize,
74                                        int compressionLevel);
75 
76 
77 
78 /* ===   Streaming functions   === */
79 
80 ZSTDMT_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
81 ZSTDMT_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);  /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
82 
83 ZSTDMT_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
84 ZSTDMT_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
85 
86 ZSTDMT_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);   /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
87 ZSTDMT_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);     /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
88 
89 
90 /* ===   Advanced functions and parameters  === */
91 
92 ZSTDMT_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
93                                           void* dst, size_t dstCapacity,
94                                     const void* src, size_t srcSize,
95                                     const ZSTD_CDict* cdict,
96                                           ZSTD_parameters params,
97                                           int overlapLog);
98 
99 ZSTDMT_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
100                                         const void* dict, size_t dictSize,   /* dict can be released after init, a local copy is preserved within zcs */
101                                         ZSTD_parameters params,
102                                         unsigned long long pledgedSrcSize);  /* pledgedSrcSize is optional and can be zero == unknown */
103 
104 ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
105                                         const ZSTD_CDict* cdict,
106                                         ZSTD_frameParameters fparams,
107                                         unsigned long long pledgedSrcSize);  /* note : zero means empty */
108 
109 /* ZSTDMT_parameter :
110  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
111 typedef enum {
112     ZSTDMT_p_jobSize,     /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
113     ZSTDMT_p_overlapLog,  /* Each job may reload a part of previous job to enhance compression ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
114     ZSTDMT_p_rsyncable    /* Enables rsyncable mode. */
115 } ZSTDMT_parameter;
116 
117 /* ZSTDMT_setMTCtxParameter() :
118  * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
119  * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
120  * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
121  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
122 ZSTDMT_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
123 
124 /* ZSTDMT_getMTCtxParameter() :
125  * Query the ZSTDMT_CCtx for a parameter value.
126  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
127 ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
128 
129 
130 /*! ZSTDMT_compressStream_generic() :
131  *  Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream()
132  *  depending on flush directive.
133  * @return : minimum amount of data still to be flushed
134  *           0 if fully flushed
135  *           or an error code
136  *  note : needs to be init using any ZSTD_initCStream*() variant */
137 ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
138                                                 ZSTD_outBuffer* output,
139                                                 ZSTD_inBuffer* input,
140                                                 ZSTD_EndDirective endOp);
141 
142 
143 /* ========================================================
144  * ===  Private interface, for use by ZSTD_compress.c   ===
145  * ===  Not exposed in libzstd. Never invoke directly   ===
146  * ======================================================== */
147 
148  /*! ZSTDMT_toFlushNow()
149   *  Tell how many bytes are ready to be flushed immediately.
150   *  Probe the oldest active job (not yet entirely flushed) and check its output buffer.
151   *  If return 0, it means there is no active job,
152   *  or, it means oldest job is still active, but everything produced has been flushed so far,
153   *  therefore flushing is limited by speed of oldest job. */
154 size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
155 
156 /*! ZSTDMT_CCtxParam_setMTCtxParameter()
157  *  like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
158 size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value);
159 
160 /*! ZSTDMT_CCtxParam_setNbWorkers()
161  *  Set nbWorkers, and clamp it.
162  *  Also reset jobSize and overlapLog */
163 size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers);
164 
165 /*! ZSTDMT_updateCParams_whileCompressing() :
166  *  Updates only a selected set of compression parameters, to remain compatible with current frame.
167  *  New parameters will be applied to next compression job. */
168 void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams);
169 
170 /*! ZSTDMT_getFrameProgression():
171  *  tells how much data has been consumed (input) and produced (output) for current frame.
172  *  able to count progression inside worker threads.
173  */
174 ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
175 
176 
177 /*! ZSTDMT_initCStream_internal() :
178  *  Private use only. Init streaming operation.
179  *  expects params to be valid.
180  *  must receive dict, or cdict, or none, but not both.
181  *  @return : 0, or an error code */
182 size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
183                     const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
184                     const ZSTD_CDict* cdict,
185                     ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
186 
187 
188 #if defined (__cplusplus)
189 }
190 #endif
191 
192 #endif   /* ZSTDMT_COMPRESS_H */
193