1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-only 2 /* 3 * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 4 * All rights reserved. 5 * 6 * This source code is licensed under both the BSD-style license (found in the 7 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 * in the COPYING file in the root directory of this source tree). 9 * You may select, at your option, one of the above-listed licenses. 10 */ 11 12 /* zstd_ddict.c : 13 * concentrates all logic that needs to know the internals of ZSTD_DDict object */ 14 15 /*-******************************************************* 16 * Dependencies 17 *********************************************************/ 18 #include <string.h> /* memcpy, memmove, memset */ 19 #include "../common/cpu.h" /* bmi2 */ 20 #include "../common/mem.h" /* low level memory routines */ 21 #define FSE_STATIC_LINKING_ONLY 22 #include "../common/fse.h" 23 #define HUF_STATIC_LINKING_ONLY 24 #include "../common/huf.h" 25 #include "zstd_decompress_internal.h" 26 #include "zstd_ddict.h" 27 28 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 29 # include "../legacy/zstd_legacy.h" 30 #endif 31 32 33 34 /*-******************************************************* 35 * Types 36 *********************************************************/ 37 struct ZSTD_DDict_s { 38 void* dictBuffer; 39 const void* dictContent; 40 size_t dictSize; 41 ZSTD_entropyDTables_t entropy; 42 U32 dictID; 43 U32 entropyPresent; 44 ZSTD_customMem cMem; 45 }; /* typedef'd to ZSTD_DDict within "zstd.h" */ 46 47 const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) 48 { 49 assert(ddict != NULL); 50 return ddict->dictContent; 51 } 52 53 size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) 54 { 55 assert(ddict != NULL); 56 return ddict->dictSize; 57 } 58 59 void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) 60 { 61 DEBUGLOG(4, "ZSTD_copyDDictParameters"); 62 assert(dctx != NULL); 63 assert(ddict != NULL); 64 dctx->dictID = ddict->dictID; 65 dctx->prefixStart = ddict->dictContent; 66 dctx->virtualStart = ddict->dictContent; 67 dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; 68 dctx->previousDstEnd = dctx->dictEnd; 69 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 70 dctx->dictContentBeginForFuzzing = dctx->prefixStart; 71 dctx->dictContentEndForFuzzing = dctx->previousDstEnd; 72 #endif 73 if (ddict->entropyPresent) { 74 dctx->litEntropy = 1; 75 dctx->fseEntropy = 1; 76 dctx->LLTptr = ddict->entropy.LLTable; 77 dctx->MLTptr = ddict->entropy.MLTable; 78 dctx->OFTptr = ddict->entropy.OFTable; 79 dctx->HUFptr = ddict->entropy.hufTable; 80 dctx->entropy.rep[0] = ddict->entropy.rep[0]; 81 dctx->entropy.rep[1] = ddict->entropy.rep[1]; 82 dctx->entropy.rep[2] = ddict->entropy.rep[2]; 83 } else { 84 dctx->litEntropy = 0; 85 dctx->fseEntropy = 0; 86 } 87 } 88 89 90 static size_t 91 ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, 92 ZSTD_dictContentType_e dictContentType) 93 { 94 ddict->dictID = 0; 95 ddict->entropyPresent = 0; 96 if (dictContentType == ZSTD_dct_rawContent) return 0; 97 98 if (ddict->dictSize < 8) { 99 if (dictContentType == ZSTD_dct_fullDict) 100 return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ 101 return 0; /* pure content mode */ 102 } 103 { U32 const magic = MEM_readLE32(ddict->dictContent); 104 if (magic != ZSTD_MAGIC_DICTIONARY) { 105 if (dictContentType == ZSTD_dct_fullDict) 106 return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ 107 return 0; /* pure content mode */ 108 } 109 } 110 ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); 111 112 /* load entropy tables */ 113 RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( 114 &ddict->entropy, ddict->dictContent, ddict->dictSize)), 115 dictionary_corrupted, ""); 116 ddict->entropyPresent = 1; 117 return 0; 118 } 119 120 121 static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, 122 const void* dict, size_t dictSize, 123 ZSTD_dictLoadMethod_e dictLoadMethod, 124 ZSTD_dictContentType_e dictContentType) 125 { 126 if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { 127 ddict->dictBuffer = NULL; 128 ddict->dictContent = dict; 129 if (!dict) dictSize = 0; 130 } else { 131 void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); 132 ddict->dictBuffer = internalBuffer; 133 ddict->dictContent = internalBuffer; 134 if (!internalBuffer) return ERROR(memory_allocation); 135 memcpy(internalBuffer, dict, dictSize); 136 } 137 ddict->dictSize = dictSize; 138 ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ 139 140 /* parse dictionary content */ 141 FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); 142 143 return 0; 144 } 145 146 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, 147 ZSTD_dictLoadMethod_e dictLoadMethod, 148 ZSTD_dictContentType_e dictContentType, 149 ZSTD_customMem customMem) 150 { 151 if (!customMem.customAlloc ^ !customMem.customFree) return NULL; 152 153 { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); 154 if (ddict == NULL) return NULL; 155 ddict->cMem = customMem; 156 { size_t const initResult = ZSTD_initDDict_internal(ddict, 157 dict, dictSize, 158 dictLoadMethod, dictContentType); 159 if (ZSTD_isError(initResult)) { 160 ZSTD_freeDDict(ddict); 161 return NULL; 162 } } 163 return ddict; 164 } 165 } 166 167 /*! ZSTD_createDDict() : 168 * Create a digested dictionary, to start decompression without startup delay. 169 * `dict` content is copied inside DDict. 170 * Consequently, `dict` can be released after `ZSTD_DDict` creation */ 171 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) 172 { 173 ZSTD_customMem const allocator = { NULL, NULL, NULL }; 174 return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); 175 } 176 177 /*! ZSTD_createDDict_byReference() : 178 * Create a digested dictionary, to start decompression without startup delay. 179 * Dictionary content is simply referenced, it will be accessed during decompression. 180 * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ 181 ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) 182 { 183 ZSTD_customMem const allocator = { NULL, NULL, NULL }; 184 return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); 185 } 186 187 188 const ZSTD_DDict* ZSTD_initStaticDDict( 189 void* sBuffer, size_t sBufferSize, 190 const void* dict, size_t dictSize, 191 ZSTD_dictLoadMethod_e dictLoadMethod, 192 ZSTD_dictContentType_e dictContentType) 193 { 194 size_t const neededSpace = sizeof(ZSTD_DDict) 195 + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); 196 ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; 197 assert(sBuffer != NULL); 198 assert(dict != NULL); 199 if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ 200 if (sBufferSize < neededSpace) return NULL; 201 if (dictLoadMethod == ZSTD_dlm_byCopy) { 202 memcpy(ddict+1, dict, dictSize); /* local copy */ 203 dict = ddict+1; 204 } 205 if (ZSTD_isError( ZSTD_initDDict_internal(ddict, 206 dict, dictSize, 207 ZSTD_dlm_byRef, dictContentType) )) 208 return NULL; 209 return ddict; 210 } 211 212 213 size_t ZSTD_freeDDict(ZSTD_DDict* ddict) 214 { 215 if (ddict==NULL) return 0; /* support free on NULL */ 216 { ZSTD_customMem const cMem = ddict->cMem; 217 ZSTD_free(ddict->dictBuffer, cMem); 218 ZSTD_free(ddict, cMem); 219 return 0; 220 } 221 } 222 223 /*! ZSTD_estimateDDictSize() : 224 * Estimate amount of memory that will be needed to create a dictionary for decompression. 225 * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ 226 size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) 227 { 228 return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); 229 } 230 231 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) 232 { 233 if (ddict==NULL) return 0; /* support sizeof on NULL */ 234 return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; 235 } 236 237 /*! ZSTD_getDictID_fromDDict() : 238 * Provides the dictID of the dictionary loaded into `ddict`. 239 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. 240 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ 241 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) 242 { 243 if (ddict==NULL) return 0; 244 return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); 245 } 246