1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-only
2 /*
3 * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
4 * All rights reserved.
5 *
6 * This source code is licensed under both the BSD-style license (found in the
7 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
8 * in the COPYING file in the root directory of this source tree).
9 * You may select, at your option, one of the above-listed licenses.
10 */
11
12 /* zstd_ddict.c :
13 * concentrates all logic that needs to know the internals of ZSTD_DDict object */
14
15 /*-*******************************************************
16 * Dependencies
17 *********************************************************/
18 #include <string.h> /* memcpy, memmove, memset */
19 #include "../common/cpu.h" /* bmi2 */
20 #include "../common/mem.h" /* low level memory routines */
21 #define FSE_STATIC_LINKING_ONLY
22 #include "../common/fse.h"
23 #define HUF_STATIC_LINKING_ONLY
24 #include "../common/huf.h"
25 #include "zstd_decompress_internal.h"
26 #include "zstd_ddict.h"
27
28 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
29 # include "../legacy/zstd_legacy.h"
30 #endif
31
32
33
34 /*-*******************************************************
35 * Types
36 *********************************************************/
37 struct ZSTD_DDict_s {
38 void* dictBuffer;
39 const void* dictContent;
40 size_t dictSize;
41 ZSTD_entropyDTables_t entropy;
42 U32 dictID;
43 U32 entropyPresent;
44 ZSTD_customMem cMem;
45 }; /* typedef'd to ZSTD_DDict within "zstd.h" */
46
ZSTD_DDict_dictContent(const ZSTD_DDict * ddict)47 const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
48 {
49 assert(ddict != NULL);
50 return ddict->dictContent;
51 }
52
ZSTD_DDict_dictSize(const ZSTD_DDict * ddict)53 size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
54 {
55 assert(ddict != NULL);
56 return ddict->dictSize;
57 }
58
ZSTD_copyDDictParameters(ZSTD_DCtx * dctx,const ZSTD_DDict * ddict)59 void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
60 {
61 DEBUGLOG(4, "ZSTD_copyDDictParameters");
62 assert(dctx != NULL);
63 assert(ddict != NULL);
64 dctx->dictID = ddict->dictID;
65 dctx->prefixStart = ddict->dictContent;
66 dctx->virtualStart = ddict->dictContent;
67 dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
68 dctx->previousDstEnd = dctx->dictEnd;
69 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
70 dctx->dictContentBeginForFuzzing = dctx->prefixStart;
71 dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
72 #endif
73 if (ddict->entropyPresent) {
74 dctx->litEntropy = 1;
75 dctx->fseEntropy = 1;
76 dctx->LLTptr = ddict->entropy.LLTable;
77 dctx->MLTptr = ddict->entropy.MLTable;
78 dctx->OFTptr = ddict->entropy.OFTable;
79 dctx->HUFptr = ddict->entropy.hufTable;
80 dctx->entropy.rep[0] = ddict->entropy.rep[0];
81 dctx->entropy.rep[1] = ddict->entropy.rep[1];
82 dctx->entropy.rep[2] = ddict->entropy.rep[2];
83 } else {
84 dctx->litEntropy = 0;
85 dctx->fseEntropy = 0;
86 }
87 }
88
89
90 static size_t
ZSTD_loadEntropy_intoDDict(ZSTD_DDict * ddict,ZSTD_dictContentType_e dictContentType)91 ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
92 ZSTD_dictContentType_e dictContentType)
93 {
94 ddict->dictID = 0;
95 ddict->entropyPresent = 0;
96 if (dictContentType == ZSTD_dct_rawContent) return 0;
97
98 if (ddict->dictSize < 8) {
99 if (dictContentType == ZSTD_dct_fullDict)
100 return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
101 return 0; /* pure content mode */
102 }
103 { U32 const magic = MEM_readLE32(ddict->dictContent);
104 if (magic != ZSTD_MAGIC_DICTIONARY) {
105 if (dictContentType == ZSTD_dct_fullDict)
106 return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
107 return 0; /* pure content mode */
108 }
109 }
110 ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
111
112 /* load entropy tables */
113 RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
114 &ddict->entropy, ddict->dictContent, ddict->dictSize)),
115 dictionary_corrupted, "");
116 ddict->entropyPresent = 1;
117 return 0;
118 }
119
120
ZSTD_initDDict_internal(ZSTD_DDict * ddict,const void * dict,size_t dictSize,ZSTD_dictLoadMethod_e dictLoadMethod,ZSTD_dictContentType_e dictContentType)121 static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
122 const void* dict, size_t dictSize,
123 ZSTD_dictLoadMethod_e dictLoadMethod,
124 ZSTD_dictContentType_e dictContentType)
125 {
126 if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
127 ddict->dictBuffer = NULL;
128 ddict->dictContent = dict;
129 if (!dict) dictSize = 0;
130 } else {
131 void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
132 ddict->dictBuffer = internalBuffer;
133 ddict->dictContent = internalBuffer;
134 if (!internalBuffer) return ERROR(memory_allocation);
135 memcpy(internalBuffer, dict, dictSize);
136 }
137 ddict->dictSize = dictSize;
138 ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
139
140 /* parse dictionary content */
141 FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
142
143 return 0;
144 }
145
ZSTD_createDDict_advanced(const void * dict,size_t dictSize,ZSTD_dictLoadMethod_e dictLoadMethod,ZSTD_dictContentType_e dictContentType,ZSTD_customMem customMem)146 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
147 ZSTD_dictLoadMethod_e dictLoadMethod,
148 ZSTD_dictContentType_e dictContentType,
149 ZSTD_customMem customMem)
150 {
151 if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
152
153 { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
154 if (ddict == NULL) return NULL;
155 ddict->cMem = customMem;
156 { size_t const initResult = ZSTD_initDDict_internal(ddict,
157 dict, dictSize,
158 dictLoadMethod, dictContentType);
159 if (ZSTD_isError(initResult)) {
160 ZSTD_freeDDict(ddict);
161 return NULL;
162 } }
163 return ddict;
164 }
165 }
166
167 /*! ZSTD_createDDict() :
168 * Create a digested dictionary, to start decompression without startup delay.
169 * `dict` content is copied inside DDict.
170 * Consequently, `dict` can be released after `ZSTD_DDict` creation */
ZSTD_createDDict(const void * dict,size_t dictSize)171 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
172 {
173 ZSTD_customMem const allocator = { NULL, NULL, NULL };
174 return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
175 }
176
177 /*! ZSTD_createDDict_byReference() :
178 * Create a digested dictionary, to start decompression without startup delay.
179 * Dictionary content is simply referenced, it will be accessed during decompression.
180 * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
ZSTD_createDDict_byReference(const void * dictBuffer,size_t dictSize)181 ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
182 {
183 ZSTD_customMem const allocator = { NULL, NULL, NULL };
184 return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
185 }
186
187
ZSTD_initStaticDDict(void * sBuffer,size_t sBufferSize,const void * dict,size_t dictSize,ZSTD_dictLoadMethod_e dictLoadMethod,ZSTD_dictContentType_e dictContentType)188 const ZSTD_DDict* ZSTD_initStaticDDict(
189 void* sBuffer, size_t sBufferSize,
190 const void* dict, size_t dictSize,
191 ZSTD_dictLoadMethod_e dictLoadMethod,
192 ZSTD_dictContentType_e dictContentType)
193 {
194 size_t const neededSpace = sizeof(ZSTD_DDict)
195 + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
196 ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
197 assert(sBuffer != NULL);
198 assert(dict != NULL);
199 if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
200 if (sBufferSize < neededSpace) return NULL;
201 if (dictLoadMethod == ZSTD_dlm_byCopy) {
202 memcpy(ddict+1, dict, dictSize); /* local copy */
203 dict = ddict+1;
204 }
205 if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
206 dict, dictSize,
207 ZSTD_dlm_byRef, dictContentType) ))
208 return NULL;
209 return ddict;
210 }
211
212
ZSTD_freeDDict(ZSTD_DDict * ddict)213 size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
214 {
215 if (ddict==NULL) return 0; /* support free on NULL */
216 { ZSTD_customMem const cMem = ddict->cMem;
217 ZSTD_free(ddict->dictBuffer, cMem);
218 ZSTD_free(ddict, cMem);
219 return 0;
220 }
221 }
222
223 /*! ZSTD_estimateDDictSize() :
224 * Estimate amount of memory that will be needed to create a dictionary for decompression.
225 * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
ZSTD_estimateDDictSize(size_t dictSize,ZSTD_dictLoadMethod_e dictLoadMethod)226 size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
227 {
228 return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
229 }
230
ZSTD_sizeof_DDict(const ZSTD_DDict * ddict)231 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
232 {
233 if (ddict==NULL) return 0; /* support sizeof on NULL */
234 return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
235 }
236
237 /*! ZSTD_getDictID_fromDDict() :
238 * Provides the dictID of the dictionary loaded into `ddict`.
239 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
240 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
ZSTD_getDictID_fromDDict(const ZSTD_DDict * ddict)241 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
242 {
243 if (ddict==NULL) return 0;
244 return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
245 }
246