1 /* 2 * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 /* zstd_ddict.c : 12 * concentrates all logic that needs to know the internals of ZSTD_DDict object */ 13 14 /*-******************************************************* 15 * Dependencies 16 *********************************************************/ 17 #include <string.h> /* memcpy, memmove, memset */ 18 #include "../common/cpu.h" /* bmi2 */ 19 #include "../common/mem.h" /* low level memory routines */ 20 #define FSE_STATIC_LINKING_ONLY 21 #include "../common/fse.h" 22 #define HUF_STATIC_LINKING_ONLY 23 #include "../common/huf.h" 24 #include "zstd_decompress_internal.h" 25 #include "zstd_ddict.h" 26 27 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 28 # include "../legacy/zstd_legacy.h" 29 #endif 30 31 32 33 /*-******************************************************* 34 * Types 35 *********************************************************/ 36 struct ZSTD_DDict_s { 37 void* dictBuffer; 38 const void* dictContent; 39 size_t dictSize; 40 ZSTD_entropyDTables_t entropy; 41 U32 dictID; 42 U32 entropyPresent; 43 ZSTD_customMem cMem; 44 }; /* typedef'd to ZSTD_DDict within "zstd.h" */ 45 46 const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) 47 { 48 assert(ddict != NULL); 49 return ddict->dictContent; 50 } 51 52 size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) 53 { 54 assert(ddict != NULL); 55 return ddict->dictSize; 56 } 57 58 void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) 59 { 60 DEBUGLOG(4, "ZSTD_copyDDictParameters"); 61 assert(dctx != NULL); 62 assert(ddict != NULL); 63 dctx->dictID = ddict->dictID; 64 dctx->prefixStart = ddict->dictContent; 65 dctx->virtualStart = ddict->dictContent; 66 dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; 67 dctx->previousDstEnd = dctx->dictEnd; 68 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 69 dctx->dictContentBeginForFuzzing = dctx->prefixStart; 70 dctx->dictContentEndForFuzzing = dctx->previousDstEnd; 71 #endif 72 if (ddict->entropyPresent) { 73 dctx->litEntropy = 1; 74 dctx->fseEntropy = 1; 75 dctx->LLTptr = ddict->entropy.LLTable; 76 dctx->MLTptr = ddict->entropy.MLTable; 77 dctx->OFTptr = ddict->entropy.OFTable; 78 dctx->HUFptr = ddict->entropy.hufTable; 79 dctx->entropy.rep[0] = ddict->entropy.rep[0]; 80 dctx->entropy.rep[1] = ddict->entropy.rep[1]; 81 dctx->entropy.rep[2] = ddict->entropy.rep[2]; 82 } else { 83 dctx->litEntropy = 0; 84 dctx->fseEntropy = 0; 85 } 86 } 87 88 89 static size_t 90 ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, 91 ZSTD_dictContentType_e dictContentType) 92 { 93 ddict->dictID = 0; 94 ddict->entropyPresent = 0; 95 if (dictContentType == ZSTD_dct_rawContent) return 0; 96 97 if (ddict->dictSize < 8) { 98 if (dictContentType == ZSTD_dct_fullDict) 99 return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ 100 return 0; /* pure content mode */ 101 } 102 { U32 const magic = MEM_readLE32(ddict->dictContent); 103 if (magic != ZSTD_MAGIC_DICTIONARY) { 104 if (dictContentType == ZSTD_dct_fullDict) 105 return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ 106 return 0; /* pure content mode */ 107 } 108 } 109 ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); 110 111 /* load entropy tables */ 112 RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( 113 &ddict->entropy, ddict->dictContent, ddict->dictSize)), 114 dictionary_corrupted, ""); 115 ddict->entropyPresent = 1; 116 return 0; 117 } 118 119 120 static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, 121 const void* dict, size_t dictSize, 122 ZSTD_dictLoadMethod_e dictLoadMethod, 123 ZSTD_dictContentType_e dictContentType) 124 { 125 if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { 126 ddict->dictBuffer = NULL; 127 ddict->dictContent = dict; 128 if (!dict) dictSize = 0; 129 } else { 130 void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); 131 ddict->dictBuffer = internalBuffer; 132 ddict->dictContent = internalBuffer; 133 if (!internalBuffer) return ERROR(memory_allocation); 134 memcpy(internalBuffer, dict, dictSize); 135 } 136 ddict->dictSize = dictSize; 137 ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ 138 139 /* parse dictionary content */ 140 FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); 141 142 return 0; 143 } 144 145 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, 146 ZSTD_dictLoadMethod_e dictLoadMethod, 147 ZSTD_dictContentType_e dictContentType, 148 ZSTD_customMem customMem) 149 { 150 if (!customMem.customAlloc ^ !customMem.customFree) return NULL; 151 152 { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); 153 if (ddict == NULL) return NULL; 154 ddict->cMem = customMem; 155 { size_t const initResult = ZSTD_initDDict_internal(ddict, 156 dict, dictSize, 157 dictLoadMethod, dictContentType); 158 if (ZSTD_isError(initResult)) { 159 ZSTD_freeDDict(ddict); 160 return NULL; 161 } } 162 return ddict; 163 } 164 } 165 166 /*! ZSTD_createDDict() : 167 * Create a digested dictionary, to start decompression without startup delay. 168 * `dict` content is copied inside DDict. 169 * Consequently, `dict` can be released after `ZSTD_DDict` creation */ 170 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) 171 { 172 ZSTD_customMem const allocator = { NULL, NULL, NULL }; 173 return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); 174 } 175 176 /*! ZSTD_createDDict_byReference() : 177 * Create a digested dictionary, to start decompression without startup delay. 178 * Dictionary content is simply referenced, it will be accessed during decompression. 179 * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ 180 ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) 181 { 182 ZSTD_customMem const allocator = { NULL, NULL, NULL }; 183 return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); 184 } 185 186 187 const ZSTD_DDict* ZSTD_initStaticDDict( 188 void* sBuffer, size_t sBufferSize, 189 const void* dict, size_t dictSize, 190 ZSTD_dictLoadMethod_e dictLoadMethod, 191 ZSTD_dictContentType_e dictContentType) 192 { 193 size_t const neededSpace = sizeof(ZSTD_DDict) 194 + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); 195 ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; 196 assert(sBuffer != NULL); 197 assert(dict != NULL); 198 if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ 199 if (sBufferSize < neededSpace) return NULL; 200 if (dictLoadMethod == ZSTD_dlm_byCopy) { 201 memcpy(ddict+1, dict, dictSize); /* local copy */ 202 dict = ddict+1; 203 } 204 if (ZSTD_isError( ZSTD_initDDict_internal(ddict, 205 dict, dictSize, 206 ZSTD_dlm_byRef, dictContentType) )) 207 return NULL; 208 return ddict; 209 } 210 211 212 size_t ZSTD_freeDDict(ZSTD_DDict* ddict) 213 { 214 if (ddict==NULL) return 0; /* support free on NULL */ 215 { ZSTD_customMem const cMem = ddict->cMem; 216 ZSTD_free(ddict->dictBuffer, cMem); 217 ZSTD_free(ddict, cMem); 218 return 0; 219 } 220 } 221 222 /*! ZSTD_estimateDDictSize() : 223 * Estimate amount of memory that will be needed to create a dictionary for decompression. 224 * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ 225 size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) 226 { 227 return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); 228 } 229 230 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) 231 { 232 if (ddict==NULL) return 0; /* support sizeof on NULL */ 233 return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; 234 } 235 236 /*! ZSTD_getDictID_fromDDict() : 237 * Provides the dictID of the dictionary loaded into `ddict`. 238 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. 239 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ 240 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) 241 { 242 if (ddict==NULL) return 0; 243 return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); 244 } 245