1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-only 2 /* 3 * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 4 * All rights reserved. 5 * 6 * This source code is licensed under both the BSD-style license (found in the 7 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 * in the COPYING file in the root directory of this source tree). 9 * You may select, at your option, one of the above-listed licenses. 10 */ 11 12 13 /* *************************************************************** 14 * Tuning parameters 15 *****************************************************************/ 16 /*! 17 * HEAPMODE : 18 * Select how default decompression function ZSTD_decompress() allocates its context, 19 * on stack (0), or into heap (1, default; requires malloc()). 20 * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. 21 */ 22 #ifndef ZSTD_HEAPMODE 23 # define ZSTD_HEAPMODE 1 24 #endif 25 26 /*! 27 * LEGACY_SUPPORT : 28 * if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) 29 */ 30 #ifndef ZSTD_LEGACY_SUPPORT 31 # define ZSTD_LEGACY_SUPPORT 0 32 #endif 33 34 /*! 35 * MAXWINDOWSIZE_DEFAULT : 36 * maximum window size accepted by DStream __by default__. 37 * Frames requiring more memory will be rejected. 38 * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). 39 */ 40 #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT 41 # define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) 42 #endif 43 44 /*! 45 * NO_FORWARD_PROGRESS_MAX : 46 * maximum allowed nb of calls to ZSTD_decompressStream() 47 * without any forward progress 48 * (defined as: no byte read from input, and no byte flushed to output) 49 * before triggering an error. 50 */ 51 #ifndef ZSTD_NO_FORWARD_PROGRESS_MAX 52 # define ZSTD_NO_FORWARD_PROGRESS_MAX 16 53 #endif 54 55 56 /*-******************************************************* 57 * Dependencies 58 *********************************************************/ 59 #include <string.h> /* memcpy, memmove, memset */ 60 #include "../common/cpu.h" /* bmi2 */ 61 #include "../common/mem.h" /* low level memory routines */ 62 #define FSE_STATIC_LINKING_ONLY 63 #include "../common/fse.h" 64 #define HUF_STATIC_LINKING_ONLY 65 #include "../common/huf.h" 66 #include "../common/zstd_internal.h" /* blockProperties_t */ 67 #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ 68 #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ 69 #include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ 70 71 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 72 # include "../legacy/zstd_legacy.h" 73 #endif 74 75 76 /*-************************************************************* 77 * Context management 78 ***************************************************************/ 79 size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) 80 { 81 if (dctx==NULL) return 0; /* support sizeof NULL */ 82 return sizeof(*dctx) 83 + ZSTD_sizeof_DDict(dctx->ddictLocal) 84 + dctx->inBuffSize + dctx->outBuffSize; 85 } 86 87 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } 88 89 90 static size_t ZSTD_startingInputLength(ZSTD_format_e format) 91 { 92 size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); 93 /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ 94 assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); 95 return startingInputLength; 96 } 97 98 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) 99 { 100 dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */ 101 dctx->staticSize = 0; 102 dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; 103 dctx->ddict = NULL; 104 dctx->ddictLocal = NULL; 105 dctx->dictEnd = NULL; 106 dctx->ddictIsCold = 0; 107 dctx->dictUses = ZSTD_dont_use; 108 dctx->inBuff = NULL; 109 dctx->inBuffSize = 0; 110 dctx->outBuffSize = 0; 111 dctx->streamStage = zdss_init; 112 dctx->legacyContext = NULL; 113 dctx->previousLegacyVersion = 0; 114 dctx->noForwardProgress = 0; 115 dctx->oversizedDuration = 0; 116 dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); 117 dctx->outBufferMode = ZSTD_obm_buffered; 118 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 119 dctx->dictContentEndForFuzzing = NULL; 120 #endif 121 } 122 123 ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) 124 { 125 ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; 126 127 if ((size_t)workspace & 7) return NULL; /* 8-aligned */ 128 if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ 129 130 ZSTD_initDCtx_internal(dctx); 131 dctx->staticSize = workspaceSize; 132 dctx->inBuff = (char*)(dctx+1); 133 return dctx; 134 } 135 136 ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) 137 { 138 if (!customMem.customAlloc ^ !customMem.customFree) return NULL; 139 140 { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem); 141 if (!dctx) return NULL; 142 dctx->customMem = customMem; 143 ZSTD_initDCtx_internal(dctx); 144 return dctx; 145 } 146 } 147 148 ZSTD_DCtx* ZSTD_createDCtx(void) 149 { 150 DEBUGLOG(3, "ZSTD_createDCtx"); 151 return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); 152 } 153 154 static void ZSTD_clearDict(ZSTD_DCtx* dctx) 155 { 156 ZSTD_freeDDict(dctx->ddictLocal); 157 dctx->ddictLocal = NULL; 158 dctx->ddict = NULL; 159 dctx->dictUses = ZSTD_dont_use; 160 } 161 162 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) 163 { 164 if (dctx==NULL) return 0; /* support free on NULL */ 165 RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); 166 { ZSTD_customMem const cMem = dctx->customMem; 167 ZSTD_clearDict(dctx); 168 ZSTD_free(dctx->inBuff, cMem); 169 dctx->inBuff = NULL; 170 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 171 if (dctx->legacyContext) 172 ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); 173 #endif 174 ZSTD_free(dctx, cMem); 175 return 0; 176 } 177 } 178 179 /* no longer useful */ 180 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) 181 { 182 size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); 183 memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ 184 } 185 186 187 /*-************************************************************* 188 * Frame header decoding 189 ***************************************************************/ 190 191 /*! ZSTD_isFrame() : 192 * Tells if the content of `buffer` starts with a valid Frame Identifier. 193 * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. 194 * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. 195 * Note 3 : Skippable Frame Identifiers are considered valid. */ 196 unsigned ZSTD_isFrame(const void* buffer, size_t size) 197 { 198 if (size < ZSTD_FRAMEIDSIZE) return 0; 199 { U32 const magic = MEM_readLE32(buffer); 200 if (magic == ZSTD_MAGICNUMBER) return 1; 201 if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; 202 } 203 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 204 if (ZSTD_isLegacy(buffer, size)) return 1; 205 #endif 206 return 0; 207 } 208 209 /** ZSTD_frameHeaderSize_internal() : 210 * srcSize must be large enough to reach header size fields. 211 * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. 212 * @return : size of the Frame Header 213 * or an error code, which can be tested with ZSTD_isError() */ 214 static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) 215 { 216 size_t const minInputSize = ZSTD_startingInputLength(format); 217 RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); 218 219 { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; 220 U32 const dictID= fhd & 3; 221 U32 const singleSegment = (fhd >> 5) & 1; 222 U32 const fcsId = fhd >> 6; 223 return minInputSize + !singleSegment 224 + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] 225 + (singleSegment && !fcsId); 226 } 227 } 228 229 /** ZSTD_frameHeaderSize() : 230 * srcSize must be >= ZSTD_frameHeaderSize_prefix. 231 * @return : size of the Frame Header, 232 * or an error code (if srcSize is too small) */ 233 size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) 234 { 235 return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); 236 } 237 238 239 /** ZSTD_getFrameHeader_advanced() : 240 * decode Frame Header, or require larger `srcSize`. 241 * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless 242 * @return : 0, `zfhPtr` is correctly filled, 243 * >0, `srcSize` is too small, value is wanted `srcSize` amount, 244 * or an error code, which can be tested using ZSTD_isError() */ 245 size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) 246 { 247 const BYTE* ip = (const BYTE*)src; 248 size_t const minInputSize = ZSTD_startingInputLength(format); 249 250 memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ 251 if (srcSize < minInputSize) return minInputSize; 252 RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); 253 254 if ( (format != ZSTD_f_zstd1_magicless) 255 && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { 256 if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 257 /* skippable frame */ 258 if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) 259 return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ 260 memset(zfhPtr, 0, sizeof(*zfhPtr)); 261 zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); 262 zfhPtr->frameType = ZSTD_skippableFrame; 263 return 0; 264 } 265 RETURN_ERROR(prefix_unknown, ""); 266 } 267 268 /* ensure there is enough `srcSize` to fully read/decode frame header */ 269 { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); 270 if (srcSize < fhsize) return fhsize; 271 zfhPtr->headerSize = (U32)fhsize; 272 } 273 274 { BYTE const fhdByte = ip[minInputSize-1]; 275 size_t pos = minInputSize; 276 U32 const dictIDSizeCode = fhdByte&3; 277 U32 const checksumFlag = (fhdByte>>2)&1; 278 U32 const singleSegment = (fhdByte>>5)&1; 279 U32 const fcsID = fhdByte>>6; 280 U64 windowSize = 0; 281 U32 dictID = 0; 282 U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; 283 RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, 284 "reserved bits, must be zero"); 285 286 if (!singleSegment) { 287 BYTE const wlByte = ip[pos++]; 288 U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; 289 RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); 290 windowSize = (1ULL << windowLog); 291 windowSize += (windowSize >> 3) * (wlByte&7); 292 } 293 switch(dictIDSizeCode) 294 { 295 default: assert(0); /* impossible */ 296 case 0 : break; 297 case 1 : dictID = ip[pos]; pos++; break; 298 case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; 299 case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; 300 } 301 switch(fcsID) 302 { 303 default: assert(0); /* impossible */ 304 case 0 : if (singleSegment) frameContentSize = ip[pos]; break; 305 case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; 306 case 2 : frameContentSize = MEM_readLE32(ip+pos); break; 307 case 3 : frameContentSize = MEM_readLE64(ip+pos); break; 308 } 309 if (singleSegment) windowSize = frameContentSize; 310 311 zfhPtr->frameType = ZSTD_frame; 312 zfhPtr->frameContentSize = frameContentSize; 313 zfhPtr->windowSize = windowSize; 314 zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); 315 zfhPtr->dictID = dictID; 316 zfhPtr->checksumFlag = checksumFlag; 317 } 318 return 0; 319 } 320 321 /** ZSTD_getFrameHeader() : 322 * decode Frame Header, or require larger `srcSize`. 323 * note : this function does not consume input, it only reads it. 324 * @return : 0, `zfhPtr` is correctly filled, 325 * >0, `srcSize` is too small, value is wanted `srcSize` amount, 326 * or an error code, which can be tested using ZSTD_isError() */ 327 size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) 328 { 329 return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); 330 } 331 332 333 /** ZSTD_getFrameContentSize() : 334 * compatible with legacy mode 335 * @return : decompressed size of the single frame pointed to be `src` if known, otherwise 336 * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined 337 * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ 338 unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) 339 { 340 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 341 if (ZSTD_isLegacy(src, srcSize)) { 342 unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); 343 return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; 344 } 345 #endif 346 { ZSTD_frameHeader zfh; 347 if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) 348 return ZSTD_CONTENTSIZE_ERROR; 349 if (zfh.frameType == ZSTD_skippableFrame) { 350 return 0; 351 } else { 352 return zfh.frameContentSize; 353 } } 354 } 355 356 static size_t readSkippableFrameSize(void const* src, size_t srcSize) 357 { 358 size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; 359 U32 sizeU32; 360 361 RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); 362 363 sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); 364 RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, 365 frameParameter_unsupported, ""); 366 { 367 size_t const skippableSize = skippableHeaderSize + sizeU32; 368 RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); 369 return skippableSize; 370 } 371 } 372 373 /** ZSTD_findDecompressedSize() : 374 * compatible with legacy mode 375 * `srcSize` must be the exact length of some number of ZSTD compressed and/or 376 * skippable frames 377 * @return : decompressed size of the frames contained */ 378 unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) 379 { 380 unsigned long long totalDstSize = 0; 381 382 while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { 383 U32 const magicNumber = MEM_readLE32(src); 384 385 if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 386 size_t const skippableSize = readSkippableFrameSize(src, srcSize); 387 if (ZSTD_isError(skippableSize)) { 388 return ZSTD_CONTENTSIZE_ERROR; 389 } 390 assert(skippableSize <= srcSize); 391 392 src = (const BYTE *)src + skippableSize; 393 srcSize -= skippableSize; 394 continue; 395 } 396 397 { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); 398 if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; 399 400 /* check for overflow */ 401 if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; 402 totalDstSize += ret; 403 } 404 { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); 405 if (ZSTD_isError(frameSrcSize)) { 406 return ZSTD_CONTENTSIZE_ERROR; 407 } 408 409 src = (const BYTE *)src + frameSrcSize; 410 srcSize -= frameSrcSize; 411 } 412 } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ 413 414 if (srcSize) return ZSTD_CONTENTSIZE_ERROR; 415 416 return totalDstSize; 417 } 418 419 /** ZSTD_getDecompressedSize() : 420 * compatible with legacy mode 421 * @return : decompressed size if known, 0 otherwise 422 note : 0 can mean any of the following : 423 - frame content is empty 424 - decompressed size field is not present in frame header 425 - frame header unknown / not supported 426 - frame header not complete (`srcSize` too small) */ 427 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) 428 { 429 unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); 430 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); 431 return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; 432 } 433 434 435 /** ZSTD_decodeFrameHeader() : 436 * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). 437 * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ 438 static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) 439 { 440 size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); 441 if (ZSTD_isError(result)) return result; /* invalid header */ 442 RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); 443 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 444 /* Skip the dictID check in fuzzing mode, because it makes the search 445 * harder. 446 */ 447 RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), 448 dictionary_wrong, ""); 449 #endif 450 if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); 451 return 0; 452 } 453 454 static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) 455 { 456 ZSTD_frameSizeInfo frameSizeInfo; 457 frameSizeInfo.compressedSize = ret; 458 frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; 459 return frameSizeInfo; 460 } 461 462 static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) 463 { 464 ZSTD_frameSizeInfo frameSizeInfo; 465 memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); 466 467 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 468 if (ZSTD_isLegacy(src, srcSize)) 469 return ZSTD_findFrameSizeInfoLegacy(src, srcSize); 470 #endif 471 472 if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) 473 && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 474 frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); 475 assert(ZSTD_isError(frameSizeInfo.compressedSize) || 476 frameSizeInfo.compressedSize <= srcSize); 477 return frameSizeInfo; 478 } else { 479 const BYTE* ip = (const BYTE*)src; 480 const BYTE* const ipstart = ip; 481 size_t remainingSize = srcSize; 482 size_t nbBlocks = 0; 483 ZSTD_frameHeader zfh; 484 485 /* Extract Frame Header */ 486 { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); 487 if (ZSTD_isError(ret)) 488 return ZSTD_errorFrameSizeInfo(ret); 489 if (ret > 0) 490 return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); 491 } 492 493 ip += zfh.headerSize; 494 remainingSize -= zfh.headerSize; 495 496 /* Iterate over each block */ 497 while (1) { 498 blockProperties_t blockProperties; 499 size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); 500 if (ZSTD_isError(cBlockSize)) 501 return ZSTD_errorFrameSizeInfo(cBlockSize); 502 503 if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) 504 return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); 505 506 ip += ZSTD_blockHeaderSize + cBlockSize; 507 remainingSize -= ZSTD_blockHeaderSize + cBlockSize; 508 nbBlocks++; 509 510 if (blockProperties.lastBlock) break; 511 } 512 513 /* Final frame content checksum */ 514 if (zfh.checksumFlag) { 515 if (remainingSize < 4) 516 return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); 517 ip += 4; 518 } 519 520 frameSizeInfo.compressedSize = ip - ipstart; 521 frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) 522 ? zfh.frameContentSize 523 : nbBlocks * zfh.blockSizeMax; 524 return frameSizeInfo; 525 } 526 } 527 528 /** ZSTD_findFrameCompressedSize() : 529 * compatible with legacy mode 530 * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame 531 * `srcSize` must be at least as large as the frame contained 532 * @return : the compressed size of the frame starting at `src` */ 533 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) 534 { 535 ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); 536 return frameSizeInfo.compressedSize; 537 } 538 539 /** ZSTD_decompressBound() : 540 * compatible with legacy mode 541 * `src` must point to the start of a ZSTD frame or a skippeable frame 542 * `srcSize` must be at least as large as the frame contained 543 * @return : the maximum decompressed size of the compressed source 544 */ 545 unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) 546 { 547 unsigned long long bound = 0; 548 /* Iterate over each frame */ 549 while (srcSize > 0) { 550 ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); 551 size_t const compressedSize = frameSizeInfo.compressedSize; 552 unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; 553 if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) 554 return ZSTD_CONTENTSIZE_ERROR; 555 assert(srcSize >= compressedSize); 556 src = (const BYTE*)src + compressedSize; 557 srcSize -= compressedSize; 558 bound += decompressedBound; 559 } 560 return bound; 561 } 562 563 564 /*-************************************************************* 565 * Frame decoding 566 ***************************************************************/ 567 568 /** ZSTD_insertBlock() : 569 * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ 570 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) 571 { 572 DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); 573 ZSTD_checkContinuity(dctx, blockStart); 574 dctx->previousDstEnd = (const char*)blockStart + blockSize; 575 return blockSize; 576 } 577 578 579 static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, 580 const void* src, size_t srcSize) 581 { 582 DEBUGLOG(5, "ZSTD_copyRawBlock"); 583 if (dst == NULL) { 584 if (srcSize == 0) return 0; 585 RETURN_ERROR(dstBuffer_null, ""); 586 } 587 RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); 588 memcpy(dst, src, srcSize); 589 return srcSize; 590 } 591 592 static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, 593 BYTE b, 594 size_t regenSize) 595 { 596 if (dst == NULL) { 597 if (regenSize == 0) return 0; 598 RETURN_ERROR(dstBuffer_null, ""); 599 } 600 RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); 601 memset(dst, b, regenSize); 602 return regenSize; 603 } 604 605 606 /*! ZSTD_decompressFrame() : 607 * @dctx must be properly initialized 608 * will update *srcPtr and *srcSizePtr, 609 * to make *srcPtr progress by one frame. */ 610 static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, 611 void* dst, size_t dstCapacity, 612 const void** srcPtr, size_t *srcSizePtr) 613 { 614 const BYTE* ip = (const BYTE*)(*srcPtr); 615 BYTE* const ostart = (BYTE* const)dst; 616 BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; 617 BYTE* op = ostart; 618 size_t remainingSrcSize = *srcSizePtr; 619 620 DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); 621 622 /* check */ 623 RETURN_ERROR_IF( 624 remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, 625 srcSize_wrong, ""); 626 627 /* Frame Header */ 628 { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( 629 ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); 630 if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; 631 RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, 632 srcSize_wrong, ""); 633 FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); 634 ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; 635 } 636 637 /* Loop on each block */ 638 while (1) { 639 size_t decodedSize; 640 blockProperties_t blockProperties; 641 size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); 642 if (ZSTD_isError(cBlockSize)) return cBlockSize; 643 644 ip += ZSTD_blockHeaderSize; 645 remainingSrcSize -= ZSTD_blockHeaderSize; 646 RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); 647 648 switch(blockProperties.blockType) 649 { 650 case bt_compressed: 651 decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize, /* frame */ 1); 652 break; 653 case bt_raw : 654 decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); 655 break; 656 case bt_rle : 657 decodedSize = ZSTD_setRleBlock(op, oend-op, *ip, blockProperties.origSize); 658 break; 659 case bt_reserved : 660 default: 661 RETURN_ERROR(corruption_detected, "invalid block type"); 662 } 663 664 if (ZSTD_isError(decodedSize)) return decodedSize; 665 if (dctx->fParams.checksumFlag) 666 XXH64_update(&dctx->xxhState, op, decodedSize); 667 if (decodedSize != 0) 668 op += decodedSize; 669 assert(ip != NULL); 670 ip += cBlockSize; 671 remainingSrcSize -= cBlockSize; 672 if (blockProperties.lastBlock) break; 673 } 674 675 if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { 676 RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, 677 corruption_detected, ""); 678 } 679 if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ 680 U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); 681 U32 checkRead; 682 RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); 683 checkRead = MEM_readLE32(ip); 684 RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); 685 ip += 4; 686 remainingSrcSize -= 4; 687 } 688 689 /* Allow caller to get size read */ 690 *srcPtr = ip; 691 *srcSizePtr = remainingSrcSize; 692 return op-ostart; 693 } 694 695 static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, 696 void* dst, size_t dstCapacity, 697 const void* src, size_t srcSize, 698 const void* dict, size_t dictSize, 699 const ZSTD_DDict* ddict) 700 { 701 void* const dststart = dst; 702 int moreThan1Frame = 0; 703 704 DEBUGLOG(5, "ZSTD_decompressMultiFrame"); 705 assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ 706 707 if (ddict) { 708 dict = ZSTD_DDict_dictContent(ddict); 709 dictSize = ZSTD_DDict_dictSize(ddict); 710 } 711 712 while (srcSize >= ZSTD_startingInputLength(dctx->format)) { 713 714 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 715 if (ZSTD_isLegacy(src, srcSize)) { 716 size_t decodedSize; 717 size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); 718 if (ZSTD_isError(frameSize)) return frameSize; 719 RETURN_ERROR_IF(dctx->staticSize, memory_allocation, 720 "legacy support is not compatible with static dctx"); 721 722 decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); 723 if (ZSTD_isError(decodedSize)) return decodedSize; 724 725 assert(decodedSize <=- dstCapacity); 726 dst = (BYTE*)dst + decodedSize; 727 dstCapacity -= decodedSize; 728 729 src = (const BYTE*)src + frameSize; 730 srcSize -= frameSize; 731 732 continue; 733 } 734 #endif 735 736 { U32 const magicNumber = MEM_readLE32(src); 737 DEBUGLOG(4, "reading magic number %08X (expecting %08X)", 738 (unsigned)magicNumber, ZSTD_MAGICNUMBER); 739 if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 740 size_t const skippableSize = readSkippableFrameSize(src, srcSize); 741 FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); 742 assert(skippableSize <= srcSize); 743 744 src = (const BYTE *)src + skippableSize; 745 srcSize -= skippableSize; 746 continue; 747 } } 748 749 if (ddict) { 750 /* we were called from ZSTD_decompress_usingDDict */ 751 FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); 752 } else { 753 /* this will initialize correctly with no dict if dict == NULL, so 754 * use this in all cases but ddict */ 755 FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); 756 } 757 ZSTD_checkContinuity(dctx, dst); 758 759 { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, 760 &src, &srcSize); 761 RETURN_ERROR_IF( 762 (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) 763 && (moreThan1Frame==1), 764 srcSize_wrong, 765 "at least one frame successfully completed, but following " 766 "bytes are garbage: it's more likely to be a srcSize error, " 767 "specifying more bytes than compressed size of frame(s). This " 768 "error message replaces ERROR(prefix_unknown), which would be " 769 "confusing, as the first header is actually correct. Note that " 770 "one could be unlucky, it might be a corruption error instead, " 771 "happening right at the place where we expect zstd magic " 772 "bytes. But this is _much_ less likely than a srcSize field " 773 "error."); 774 if (ZSTD_isError(res)) return res; 775 assert(res <= dstCapacity); 776 if (res != 0) 777 dst = (BYTE*)dst + res; 778 dstCapacity -= res; 779 } 780 moreThan1Frame = 1; 781 } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ 782 783 RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); 784 785 return (BYTE*)dst - (BYTE*)dststart; 786 } 787 788 size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, 789 void* dst, size_t dstCapacity, 790 const void* src, size_t srcSize, 791 const void* dict, size_t dictSize) 792 { 793 return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); 794 } 795 796 797 static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) 798 { 799 switch (dctx->dictUses) { 800 default: 801 assert(0 /* Impossible */); 802 /* fall-through */ 803 case ZSTD_dont_use: 804 ZSTD_clearDict(dctx); 805 return NULL; 806 case ZSTD_use_indefinitely: 807 return dctx->ddict; 808 case ZSTD_use_once: 809 dctx->dictUses = ZSTD_dont_use; 810 return dctx->ddict; 811 } 812 } 813 814 size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) 815 { 816 return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); 817 } 818 819 820 size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) 821 { 822 #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) 823 size_t regenSize; 824 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 825 RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); 826 regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); 827 ZSTD_freeDCtx(dctx); 828 return regenSize; 829 #else /* stack mode */ 830 ZSTD_DCtx dctx; 831 ZSTD_initDCtx_internal(&dctx); 832 return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); 833 #endif 834 } 835 836 837 /*-************************************** 838 * Advanced Streaming Decompression API 839 * Bufferless and synchronous 840 ****************************************/ 841 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } 842 843 /** 844 * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, 845 * we allow taking a partial block as the input. Currently only raw uncompressed blocks can 846 * be streamed. 847 * 848 * For blocks that can be streamed, this allows us to reduce the latency until we produce 849 * output, and avoid copying the input. 850 * 851 * @param inputSize - The total amount of input that the caller currently has. 852 */ 853 static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { 854 if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) 855 return dctx->expected; 856 if (dctx->bType != bt_raw) 857 return dctx->expected; 858 return MIN(MAX(inputSize, 1), dctx->expected); 859 } 860 861 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { 862 switch(dctx->stage) 863 { 864 default: /* should not happen */ 865 assert(0); 866 case ZSTDds_getFrameHeaderSize: 867 case ZSTDds_decodeFrameHeader: 868 return ZSTDnit_frameHeader; 869 case ZSTDds_decodeBlockHeader: 870 return ZSTDnit_blockHeader; 871 case ZSTDds_decompressBlock: 872 return ZSTDnit_block; 873 case ZSTDds_decompressLastBlock: 874 return ZSTDnit_lastBlock; 875 case ZSTDds_checkChecksum: 876 return ZSTDnit_checksum; 877 case ZSTDds_decodeSkippableHeader: 878 case ZSTDds_skipFrame: 879 return ZSTDnit_skippableFrame; 880 } 881 } 882 883 static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } 884 885 /** ZSTD_decompressContinue() : 886 * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) 887 * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) 888 * or an error code, which can be tested using ZSTD_isError() */ 889 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) 890 { 891 DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); 892 /* Sanity check */ 893 RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); 894 if (dstCapacity) ZSTD_checkContinuity(dctx, dst); 895 896 switch (dctx->stage) 897 { 898 case ZSTDds_getFrameHeaderSize : 899 assert(src != NULL); 900 if (dctx->format == ZSTD_f_zstd1) { /* allows header */ 901 assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ 902 if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ 903 memcpy(dctx->headerBuffer, src, srcSize); 904 dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ 905 dctx->stage = ZSTDds_decodeSkippableHeader; 906 return 0; 907 } } 908 dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); 909 if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; 910 memcpy(dctx->headerBuffer, src, srcSize); 911 dctx->expected = dctx->headerSize - srcSize; 912 dctx->stage = ZSTDds_decodeFrameHeader; 913 return 0; 914 915 case ZSTDds_decodeFrameHeader: 916 assert(src != NULL); 917 memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); 918 FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); 919 dctx->expected = ZSTD_blockHeaderSize; 920 dctx->stage = ZSTDds_decodeBlockHeader; 921 return 0; 922 923 case ZSTDds_decodeBlockHeader: 924 { blockProperties_t bp; 925 size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); 926 if (ZSTD_isError(cBlockSize)) return cBlockSize; 927 RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); 928 dctx->expected = cBlockSize; 929 dctx->bType = bp.blockType; 930 dctx->rleSize = bp.origSize; 931 if (cBlockSize) { 932 dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; 933 return 0; 934 } 935 /* empty block */ 936 if (bp.lastBlock) { 937 if (dctx->fParams.checksumFlag) { 938 dctx->expected = 4; 939 dctx->stage = ZSTDds_checkChecksum; 940 } else { 941 dctx->expected = 0; /* end of frame */ 942 dctx->stage = ZSTDds_getFrameHeaderSize; 943 } 944 } else { 945 dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ 946 dctx->stage = ZSTDds_decodeBlockHeader; 947 } 948 return 0; 949 } 950 951 case ZSTDds_decompressLastBlock: 952 case ZSTDds_decompressBlock: 953 DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); 954 { size_t rSize; 955 switch(dctx->bType) 956 { 957 case bt_compressed: 958 DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); 959 rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); 960 dctx->expected = 0; /* Streaming not supported */ 961 break; 962 case bt_raw : 963 assert(srcSize <= dctx->expected); 964 rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); 965 FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); 966 assert(rSize == srcSize); 967 dctx->expected -= rSize; 968 break; 969 case bt_rle : 970 rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); 971 dctx->expected = 0; /* Streaming not supported */ 972 break; 973 case bt_reserved : /* should never happen */ 974 default: 975 RETURN_ERROR(corruption_detected, "invalid block type"); 976 } 977 FORWARD_IF_ERROR(rSize, ""); 978 RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); 979 DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); 980 dctx->decodedSize += rSize; 981 if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); 982 dctx->previousDstEnd = (char*)dst + rSize; 983 984 /* Stay on the same stage until we are finished streaming the block. */ 985 if (dctx->expected > 0) { 986 return rSize; 987 } 988 989 if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ 990 DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); 991 RETURN_ERROR_IF( 992 dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN 993 && dctx->decodedSize != dctx->fParams.frameContentSize, 994 corruption_detected, ""); 995 if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ 996 dctx->expected = 4; 997 dctx->stage = ZSTDds_checkChecksum; 998 } else { 999 dctx->expected = 0; /* ends here */ 1000 dctx->stage = ZSTDds_getFrameHeaderSize; 1001 } 1002 } else { 1003 dctx->stage = ZSTDds_decodeBlockHeader; 1004 dctx->expected = ZSTD_blockHeaderSize; 1005 } 1006 return rSize; 1007 } 1008 1009 case ZSTDds_checkChecksum: 1010 assert(srcSize == 4); /* guaranteed by dctx->expected */ 1011 { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); 1012 U32 const check32 = MEM_readLE32(src); 1013 DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); 1014 RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); 1015 dctx->expected = 0; 1016 dctx->stage = ZSTDds_getFrameHeaderSize; 1017 return 0; 1018 } 1019 1020 case ZSTDds_decodeSkippableHeader: 1021 assert(src != NULL); 1022 assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); 1023 memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ 1024 dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ 1025 dctx->stage = ZSTDds_skipFrame; 1026 return 0; 1027 1028 case ZSTDds_skipFrame: 1029 dctx->expected = 0; 1030 dctx->stage = ZSTDds_getFrameHeaderSize; 1031 return 0; 1032 1033 default: 1034 assert(0); /* impossible */ 1035 RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ 1036 } 1037 } 1038 1039 1040 static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1041 { 1042 dctx->dictEnd = dctx->previousDstEnd; 1043 dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); 1044 dctx->prefixStart = dict; 1045 dctx->previousDstEnd = (const char*)dict + dictSize; 1046 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 1047 dctx->dictContentBeginForFuzzing = dctx->prefixStart; 1048 dctx->dictContentEndForFuzzing = dctx->previousDstEnd; 1049 #endif 1050 return 0; 1051 } 1052 1053 /*! ZSTD_loadDEntropy() : 1054 * dict : must point at beginning of a valid zstd dictionary. 1055 * @return : size of entropy tables read */ 1056 size_t 1057 ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, 1058 const void* const dict, size_t const dictSize) 1059 { 1060 const BYTE* dictPtr = (const BYTE*)dict; 1061 const BYTE* const dictEnd = dictPtr + dictSize; 1062 1063 RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); 1064 assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ 1065 dictPtr += 8; /* skip header = magic + dictID */ 1066 1067 ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); 1068 ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); 1069 ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); 1070 { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ 1071 size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); 1072 #ifdef HUF_FORCE_DECOMPRESS_X1 1073 /* in minimal huffman, we always use X1 variants */ 1074 size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, 1075 dictPtr, dictEnd - dictPtr, 1076 workspace, workspaceSize); 1077 #else 1078 size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, 1079 dictPtr, dictEnd - dictPtr, 1080 workspace, workspaceSize); 1081 #endif 1082 RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); 1083 dictPtr += hSize; 1084 } 1085 1086 { short offcodeNCount[MaxOff+1]; 1087 unsigned offcodeMaxValue = MaxOff, offcodeLog; 1088 size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); 1089 RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); 1090 RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); 1091 RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); 1092 ZSTD_buildFSETable( entropy->OFTable, 1093 offcodeNCount, offcodeMaxValue, 1094 OF_base, OF_bits, 1095 offcodeLog); 1096 dictPtr += offcodeHeaderSize; 1097 } 1098 1099 { short matchlengthNCount[MaxML+1]; 1100 unsigned matchlengthMaxValue = MaxML, matchlengthLog; 1101 size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); 1102 RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); 1103 RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); 1104 RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); 1105 ZSTD_buildFSETable( entropy->MLTable, 1106 matchlengthNCount, matchlengthMaxValue, 1107 ML_base, ML_bits, 1108 matchlengthLog); 1109 dictPtr += matchlengthHeaderSize; 1110 } 1111 1112 { short litlengthNCount[MaxLL+1]; 1113 unsigned litlengthMaxValue = MaxLL, litlengthLog; 1114 size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); 1115 RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); 1116 RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); 1117 RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); 1118 ZSTD_buildFSETable( entropy->LLTable, 1119 litlengthNCount, litlengthMaxValue, 1120 LL_base, LL_bits, 1121 litlengthLog); 1122 dictPtr += litlengthHeaderSize; 1123 } 1124 1125 RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); 1126 { int i; 1127 size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); 1128 for (i=0; i<3; i++) { 1129 U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; 1130 RETURN_ERROR_IF(rep==0 || rep > dictContentSize, 1131 dictionary_corrupted, ""); 1132 entropy->rep[i] = rep; 1133 } } 1134 1135 return dictPtr - (const BYTE*)dict; 1136 } 1137 1138 static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1139 { 1140 if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); 1141 { U32 const magic = MEM_readLE32(dict); 1142 if (magic != ZSTD_MAGIC_DICTIONARY) { 1143 return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ 1144 } } 1145 dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); 1146 1147 /* load entropy tables */ 1148 { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); 1149 RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); 1150 dict = (const char*)dict + eSize; 1151 dictSize -= eSize; 1152 } 1153 dctx->litEntropy = dctx->fseEntropy = 1; 1154 1155 /* reference dictionary content */ 1156 return ZSTD_refDictContent(dctx, dict, dictSize); 1157 } 1158 1159 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) 1160 { 1161 assert(dctx != NULL); 1162 dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ 1163 dctx->stage = ZSTDds_getFrameHeaderSize; 1164 dctx->decodedSize = 0; 1165 dctx->previousDstEnd = NULL; 1166 dctx->prefixStart = NULL; 1167 dctx->virtualStart = NULL; 1168 dctx->dictEnd = NULL; 1169 dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ 1170 dctx->litEntropy = dctx->fseEntropy = 0; 1171 dctx->dictID = 0; 1172 dctx->bType = bt_reserved; 1173 ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); 1174 memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ 1175 dctx->LLTptr = dctx->entropy.LLTable; 1176 dctx->MLTptr = dctx->entropy.MLTable; 1177 dctx->OFTptr = dctx->entropy.OFTable; 1178 dctx->HUFptr = dctx->entropy.hufTable; 1179 return 0; 1180 } 1181 1182 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1183 { 1184 FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); 1185 if (dict && dictSize) 1186 RETURN_ERROR_IF( 1187 ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), 1188 dictionary_corrupted, ""); 1189 return 0; 1190 } 1191 1192 1193 /* ====== ZSTD_DDict ====== */ 1194 1195 size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) 1196 { 1197 DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); 1198 assert(dctx != NULL); 1199 if (ddict) { 1200 const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); 1201 size_t const dictSize = ZSTD_DDict_dictSize(ddict); 1202 const void* const dictEnd = dictStart + dictSize; 1203 dctx->ddictIsCold = (dctx->dictEnd != dictEnd); 1204 DEBUGLOG(4, "DDict is %s", 1205 dctx->ddictIsCold ? "~cold~" : "hot!"); 1206 } 1207 FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); 1208 if (ddict) { /* NULL ddict is equivalent to no dictionary */ 1209 ZSTD_copyDDictParameters(dctx, ddict); 1210 } 1211 return 0; 1212 } 1213 1214 /*! ZSTD_getDictID_fromDict() : 1215 * Provides the dictID stored within dictionary. 1216 * if @return == 0, the dictionary is not conformant with Zstandard specification. 1217 * It can still be loaded, but as a content-only dictionary. */ 1218 unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) 1219 { 1220 if (dictSize < 8) return 0; 1221 if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; 1222 return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); 1223 } 1224 1225 /*! ZSTD_getDictID_fromFrame() : 1226 * Provides the dictID required to decompress frame stored within `src`. 1227 * If @return == 0, the dictID could not be decoded. 1228 * This could for one of the following reasons : 1229 * - The frame does not require a dictionary (most common case). 1230 * - The frame was built with dictID intentionally removed. 1231 * Needed dictionary is a hidden information. 1232 * Note : this use case also happens when using a non-conformant dictionary. 1233 * - `srcSize` is too small, and as a result, frame header could not be decoded. 1234 * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. 1235 * - This is not a Zstandard frame. 1236 * When identifying the exact failure cause, it's possible to use 1237 * ZSTD_getFrameHeader(), which will provide a more precise error code. */ 1238 unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) 1239 { 1240 ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; 1241 size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); 1242 if (ZSTD_isError(hError)) return 0; 1243 return zfp.dictID; 1244 } 1245 1246 1247 /*! ZSTD_decompress_usingDDict() : 1248 * Decompression using a pre-digested Dictionary 1249 * Use dictionary without significant overhead. */ 1250 size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, 1251 void* dst, size_t dstCapacity, 1252 const void* src, size_t srcSize, 1253 const ZSTD_DDict* ddict) 1254 { 1255 /* pass content and size in case legacy frames are encountered */ 1256 return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, 1257 NULL, 0, 1258 ddict); 1259 } 1260 1261 1262 /*===================================== 1263 * Streaming decompression 1264 *====================================*/ 1265 1266 ZSTD_DStream* ZSTD_createDStream(void) 1267 { 1268 DEBUGLOG(3, "ZSTD_createDStream"); 1269 return ZSTD_createDStream_advanced(ZSTD_defaultCMem); 1270 } 1271 1272 ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) 1273 { 1274 return ZSTD_initStaticDCtx(workspace, workspaceSize); 1275 } 1276 1277 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) 1278 { 1279 return ZSTD_createDCtx_advanced(customMem); 1280 } 1281 1282 size_t ZSTD_freeDStream(ZSTD_DStream* zds) 1283 { 1284 return ZSTD_freeDCtx(zds); 1285 } 1286 1287 1288 /* *** Initialization *** */ 1289 1290 size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } 1291 size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } 1292 1293 size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, 1294 const void* dict, size_t dictSize, 1295 ZSTD_dictLoadMethod_e dictLoadMethod, 1296 ZSTD_dictContentType_e dictContentType) 1297 { 1298 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); 1299 ZSTD_clearDict(dctx); 1300 if (dict && dictSize != 0) { 1301 dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); 1302 RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); 1303 dctx->ddict = dctx->ddictLocal; 1304 dctx->dictUses = ZSTD_use_indefinitely; 1305 } 1306 return 0; 1307 } 1308 1309 size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1310 { 1311 return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); 1312 } 1313 1314 size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1315 { 1316 return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); 1317 } 1318 1319 size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) 1320 { 1321 FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); 1322 dctx->dictUses = ZSTD_use_once; 1323 return 0; 1324 } 1325 1326 size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) 1327 { 1328 return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); 1329 } 1330 1331 1332 /* ZSTD_initDStream_usingDict() : 1333 * return : expected size, aka ZSTD_startingInputLength(). 1334 * this function cannot fail */ 1335 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) 1336 { 1337 DEBUGLOG(4, "ZSTD_initDStream_usingDict"); 1338 FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); 1339 FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); 1340 return ZSTD_startingInputLength(zds->format); 1341 } 1342 1343 /* note : this variant can't fail */ 1344 size_t ZSTD_initDStream(ZSTD_DStream* zds) 1345 { 1346 DEBUGLOG(4, "ZSTD_initDStream"); 1347 return ZSTD_initDStream_usingDDict(zds, NULL); 1348 } 1349 1350 /* ZSTD_initDStream_usingDDict() : 1351 * ddict will just be referenced, and must outlive decompression session 1352 * this function cannot fail */ 1353 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) 1354 { 1355 FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); 1356 FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); 1357 return ZSTD_startingInputLength(dctx->format); 1358 } 1359 1360 /* ZSTD_resetDStream() : 1361 * return : expected size, aka ZSTD_startingInputLength(). 1362 * this function cannot fail */ 1363 size_t ZSTD_resetDStream(ZSTD_DStream* dctx) 1364 { 1365 FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); 1366 return ZSTD_startingInputLength(dctx->format); 1367 } 1368 1369 1370 size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) 1371 { 1372 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); 1373 ZSTD_clearDict(dctx); 1374 if (ddict) { 1375 dctx->ddict = ddict; 1376 dctx->dictUses = ZSTD_use_indefinitely; 1377 } 1378 return 0; 1379 } 1380 1381 /* ZSTD_DCtx_setMaxWindowSize() : 1382 * note : no direct equivalence in ZSTD_DCtx_setParameter, 1383 * since this version sets windowSize, and the other sets windowLog */ 1384 size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) 1385 { 1386 ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); 1387 size_t const min = (size_t)1 << bounds.lowerBound; 1388 size_t const max = (size_t)1 << bounds.upperBound; 1389 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); 1390 RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); 1391 RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); 1392 dctx->maxWindowSize = maxWindowSize; 1393 return 0; 1394 } 1395 1396 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) 1397 { 1398 return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format); 1399 } 1400 1401 ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) 1402 { 1403 ZSTD_bounds bounds = { 0, 0, 0 }; 1404 switch(dParam) { 1405 case ZSTD_d_windowLogMax: 1406 bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; 1407 bounds.upperBound = ZSTD_WINDOWLOG_MAX; 1408 return bounds; 1409 case ZSTD_d_format: 1410 bounds.lowerBound = (int)ZSTD_f_zstd1; 1411 bounds.upperBound = (int)ZSTD_f_zstd1_magicless; 1412 ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); 1413 return bounds; 1414 case ZSTD_d_stableOutBuffer: 1415 bounds.lowerBound = (int)ZSTD_obm_buffered; 1416 bounds.upperBound = (int)ZSTD_obm_stable; 1417 return bounds; 1418 default:; 1419 } 1420 bounds.error = ERROR(parameter_unsupported); 1421 return bounds; 1422 } 1423 1424 /* ZSTD_dParam_withinBounds: 1425 * @return 1 if value is within dParam bounds, 1426 * 0 otherwise */ 1427 static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) 1428 { 1429 ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); 1430 if (ZSTD_isError(bounds.error)) return 0; 1431 if (value < bounds.lowerBound) return 0; 1432 if (value > bounds.upperBound) return 0; 1433 return 1; 1434 } 1435 1436 #define CHECK_DBOUNDS(p,v) { \ 1437 RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ 1438 } 1439 1440 size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) 1441 { 1442 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); 1443 switch(dParam) { 1444 case ZSTD_d_windowLogMax: 1445 if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; 1446 CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); 1447 dctx->maxWindowSize = ((size_t)1) << value; 1448 return 0; 1449 case ZSTD_d_format: 1450 CHECK_DBOUNDS(ZSTD_d_format, value); 1451 dctx->format = (ZSTD_format_e)value; 1452 return 0; 1453 case ZSTD_d_stableOutBuffer: 1454 CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); 1455 dctx->outBufferMode = (ZSTD_outBufferMode_e)value; 1456 return 0; 1457 default:; 1458 } 1459 RETURN_ERROR(parameter_unsupported, ""); 1460 } 1461 1462 size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) 1463 { 1464 if ( (reset == ZSTD_reset_session_only) 1465 || (reset == ZSTD_reset_session_and_parameters) ) { 1466 dctx->streamStage = zdss_init; 1467 dctx->noForwardProgress = 0; 1468 } 1469 if ( (reset == ZSTD_reset_parameters) 1470 || (reset == ZSTD_reset_session_and_parameters) ) { 1471 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); 1472 ZSTD_clearDict(dctx); 1473 dctx->format = ZSTD_f_zstd1; 1474 dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; 1475 } 1476 return 0; 1477 } 1478 1479 1480 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) 1481 { 1482 return ZSTD_sizeof_DCtx(dctx); 1483 } 1484 1485 size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) 1486 { 1487 size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); 1488 unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); 1489 unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); 1490 size_t const minRBSize = (size_t) neededSize; 1491 RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, 1492 frameParameter_windowTooLarge, ""); 1493 return minRBSize; 1494 } 1495 1496 size_t ZSTD_estimateDStreamSize(size_t windowSize) 1497 { 1498 size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); 1499 size_t const inBuffSize = blockSize; /* no block can be larger */ 1500 size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); 1501 return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; 1502 } 1503 1504 size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) 1505 { 1506 U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ 1507 ZSTD_frameHeader zfh; 1508 size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); 1509 if (ZSTD_isError(err)) return err; 1510 RETURN_ERROR_IF(err>0, srcSize_wrong, ""); 1511 RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, 1512 frameParameter_windowTooLarge, ""); 1513 return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); 1514 } 1515 1516 1517 /* ***** Decompression ***** */ 1518 1519 static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) 1520 { 1521 return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; 1522 } 1523 1524 static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) 1525 { 1526 if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) 1527 zds->oversizedDuration++; 1528 else 1529 zds->oversizedDuration = 0; 1530 } 1531 1532 static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) 1533 { 1534 return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; 1535 } 1536 1537 /* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ 1538 static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) 1539 { 1540 ZSTD_outBuffer const expect = zds->expectedOutBuffer; 1541 /* No requirement when ZSTD_obm_stable is not enabled. */ 1542 if (zds->outBufferMode != ZSTD_obm_stable) 1543 return 0; 1544 /* Any buffer is allowed in zdss_init, this must be the same for every other call until 1545 * the context is reset. 1546 */ 1547 if (zds->streamStage == zdss_init) 1548 return 0; 1549 /* The buffer must match our expectation exactly. */ 1550 if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) 1551 return 0; 1552 RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!"); 1553 } 1554 1555 /* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() 1556 * and updates the stage and the output buffer state. This call is extracted so it can be 1557 * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. 1558 * NOTE: You must break after calling this function since the streamStage is modified. 1559 */ 1560 static size_t ZSTD_decompressContinueStream( 1561 ZSTD_DStream* zds, char** op, char* oend, 1562 void const* src, size_t srcSize) { 1563 int const isSkipFrame = ZSTD_isSkipFrame(zds); 1564 if (zds->outBufferMode == ZSTD_obm_buffered) { 1565 size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; 1566 size_t const decodedSize = ZSTD_decompressContinue(zds, 1567 zds->outBuff + zds->outStart, dstSize, src, srcSize); 1568 FORWARD_IF_ERROR(decodedSize, ""); 1569 if (!decodedSize && !isSkipFrame) { 1570 zds->streamStage = zdss_read; 1571 } else { 1572 zds->outEnd = zds->outStart + decodedSize; 1573 zds->streamStage = zdss_flush; 1574 } 1575 } else { 1576 /* Write directly into the output buffer */ 1577 size_t const dstSize = isSkipFrame ? 0 : oend - *op; 1578 size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); 1579 FORWARD_IF_ERROR(decodedSize, ""); 1580 *op += decodedSize; 1581 /* Flushing is not needed. */ 1582 zds->streamStage = zdss_read; 1583 assert(*op <= oend); 1584 assert(zds->outBufferMode == ZSTD_obm_stable); 1585 } 1586 return 0; 1587 } 1588 1589 size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) 1590 { 1591 const char* const src = (const char*)input->src; 1592 const char* const istart = input->pos != 0 ? src + input->pos : src; 1593 const char* const iend = input->size != 0 ? src + input->size : src; 1594 const char* ip = istart; 1595 char* const dst = (char*)output->dst; 1596 char* const ostart = output->pos != 0 ? dst + output->pos : dst; 1597 char* const oend = output->size != 0 ? dst + output->size : dst; 1598 char* op = ostart; 1599 U32 someMoreWork = 1; 1600 1601 DEBUGLOG(5, "ZSTD_decompressStream"); 1602 RETURN_ERROR_IF( 1603 input->pos > input->size, 1604 srcSize_wrong, 1605 "forbidden. in: pos: %u vs size: %u", 1606 (U32)input->pos, (U32)input->size); 1607 RETURN_ERROR_IF( 1608 output->pos > output->size, 1609 dstSize_tooSmall, 1610 "forbidden. out: pos: %u vs size: %u", 1611 (U32)output->pos, (U32)output->size); 1612 DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); 1613 FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); 1614 1615 while (someMoreWork) { 1616 switch(zds->streamStage) 1617 { 1618 case zdss_init : 1619 DEBUGLOG(5, "stage zdss_init => transparent reset "); 1620 zds->streamStage = zdss_loadHeader; 1621 zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; 1622 zds->legacyVersion = 0; 1623 zds->hostageByte = 0; 1624 zds->expectedOutBuffer = *output; 1625 /* fall-through */ 1626 1627 case zdss_loadHeader : 1628 DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); 1629 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 1630 if (zds->legacyVersion) { 1631 RETURN_ERROR_IF(zds->staticSize, memory_allocation, 1632 "legacy support is incompatible with static dctx"); 1633 { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); 1634 if (hint==0) zds->streamStage = zdss_init; 1635 return hint; 1636 } } 1637 #endif 1638 { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); 1639 DEBUGLOG(5, "header size : %u", (U32)hSize); 1640 if (ZSTD_isError(hSize)) { 1641 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 1642 U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); 1643 if (legacyVersion) { 1644 ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); 1645 const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; 1646 size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; 1647 DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); 1648 RETURN_ERROR_IF(zds->staticSize, memory_allocation, 1649 "legacy support is incompatible with static dctx"); 1650 FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, 1651 zds->previousLegacyVersion, legacyVersion, 1652 dict, dictSize), ""); 1653 zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; 1654 { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); 1655 if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ 1656 return hint; 1657 } } 1658 #endif 1659 return hSize; /* error */ 1660 } 1661 if (hSize != 0) { /* need more input */ 1662 size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ 1663 size_t const remainingInput = (size_t)(iend-ip); 1664 assert(iend >= ip); 1665 if (toLoad > remainingInput) { /* not enough input to load full header */ 1666 if (remainingInput > 0) { 1667 memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); 1668 zds->lhSize += remainingInput; 1669 } 1670 input->pos = input->size; 1671 return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ 1672 } 1673 assert(ip != NULL); 1674 memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; 1675 break; 1676 } } 1677 1678 /* check for single-pass mode opportunity */ 1679 if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN 1680 && zds->fParams.frameType != ZSTD_skippableFrame 1681 && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { 1682 size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); 1683 if (cSize <= (size_t)(iend-istart)) { 1684 /* shortcut : using single-pass mode */ 1685 size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds)); 1686 if (ZSTD_isError(decompressedSize)) return decompressedSize; 1687 DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") 1688 ip = istart + cSize; 1689 op += decompressedSize; 1690 zds->expected = 0; 1691 zds->streamStage = zdss_init; 1692 someMoreWork = 0; 1693 break; 1694 } } 1695 1696 /* Check output buffer is large enough for ZSTD_odm_stable. */ 1697 if (zds->outBufferMode == ZSTD_obm_stable 1698 && zds->fParams.frameType != ZSTD_skippableFrame 1699 && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN 1700 && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { 1701 RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); 1702 } 1703 1704 /* Consume header (see ZSTDds_decodeFrameHeader) */ 1705 DEBUGLOG(4, "Consume header"); 1706 FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); 1707 1708 if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ 1709 zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); 1710 zds->stage = ZSTDds_skipFrame; 1711 } else { 1712 FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); 1713 zds->expected = ZSTD_blockHeaderSize; 1714 zds->stage = ZSTDds_decodeBlockHeader; 1715 } 1716 1717 /* control buffer memory usage */ 1718 DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", 1719 (U32)(zds->fParams.windowSize >>10), 1720 (U32)(zds->maxWindowSize >> 10) ); 1721 zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); 1722 RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, 1723 frameParameter_windowTooLarge, ""); 1724 1725 /* Adapt buffer sizes to frame header instructions */ 1726 { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); 1727 size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered 1728 ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) 1729 : 0; 1730 1731 ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); 1732 1733 { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); 1734 int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); 1735 1736 if (tooSmall || tooLarge) { 1737 size_t const bufferSize = neededInBuffSize + neededOutBuffSize; 1738 DEBUGLOG(4, "inBuff : from %u to %u", 1739 (U32)zds->inBuffSize, (U32)neededInBuffSize); 1740 DEBUGLOG(4, "outBuff : from %u to %u", 1741 (U32)zds->outBuffSize, (U32)neededOutBuffSize); 1742 if (zds->staticSize) { /* static DCtx */ 1743 DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); 1744 assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ 1745 RETURN_ERROR_IF( 1746 bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), 1747 memory_allocation, ""); 1748 } else { 1749 ZSTD_free(zds->inBuff, zds->customMem); 1750 zds->inBuffSize = 0; 1751 zds->outBuffSize = 0; 1752 zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); 1753 RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); 1754 } 1755 zds->inBuffSize = neededInBuffSize; 1756 zds->outBuff = zds->inBuff + zds->inBuffSize; 1757 zds->outBuffSize = neededOutBuffSize; 1758 } } } 1759 zds->streamStage = zdss_read; 1760 /* fall-through */ 1761 1762 case zdss_read: 1763 DEBUGLOG(5, "stage zdss_read"); 1764 { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip); 1765 DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); 1766 if (neededInSize==0) { /* end of frame */ 1767 zds->streamStage = zdss_init; 1768 someMoreWork = 0; 1769 break; 1770 } 1771 if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ 1772 FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); 1773 ip += neededInSize; 1774 /* Function modifies the stage so we must break */ 1775 break; 1776 } } 1777 if (ip==iend) { someMoreWork = 0; break; } /* no more input */ 1778 zds->streamStage = zdss_load; 1779 /* fall-through */ 1780 1781 case zdss_load: 1782 { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); 1783 size_t const toLoad = neededInSize - zds->inPos; 1784 int const isSkipFrame = ZSTD_isSkipFrame(zds); 1785 size_t loadedSize; 1786 /* At this point we shouldn't be decompressing a block that we can stream. */ 1787 assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); 1788 if (isSkipFrame) { 1789 loadedSize = MIN(toLoad, (size_t)(iend-ip)); 1790 } else { 1791 RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, 1792 corruption_detected, 1793 "should never happen"); 1794 loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); 1795 } 1796 ip += loadedSize; 1797 zds->inPos += loadedSize; 1798 if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ 1799 1800 /* decode loaded input */ 1801 zds->inPos = 0; /* input is consumed */ 1802 FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); 1803 /* Function modifies the stage so we must break */ 1804 break; 1805 } 1806 case zdss_flush: 1807 { size_t const toFlushSize = zds->outEnd - zds->outStart; 1808 size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); 1809 op += flushedSize; 1810 zds->outStart += flushedSize; 1811 if (flushedSize == toFlushSize) { /* flush completed */ 1812 zds->streamStage = zdss_read; 1813 if ( (zds->outBuffSize < zds->fParams.frameContentSize) 1814 && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { 1815 DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", 1816 (int)(zds->outBuffSize - zds->outStart), 1817 (U32)zds->fParams.blockSizeMax); 1818 zds->outStart = zds->outEnd = 0; 1819 } 1820 break; 1821 } } 1822 /* cannot complete flush */ 1823 someMoreWork = 0; 1824 break; 1825 1826 default: 1827 assert(0); /* impossible */ 1828 RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ 1829 } } 1830 1831 /* result */ 1832 input->pos = (size_t)(ip - (const char*)(input->src)); 1833 output->pos = (size_t)(op - (char*)(output->dst)); 1834 1835 /* Update the expected output buffer for ZSTD_obm_stable. */ 1836 zds->expectedOutBuffer = *output; 1837 1838 if ((ip==istart) && (op==ostart)) { /* no forward progress */ 1839 zds->noForwardProgress ++; 1840 if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { 1841 RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); 1842 RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); 1843 assert(0); 1844 } 1845 } else { 1846 zds->noForwardProgress = 0; 1847 } 1848 { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); 1849 if (!nextSrcSizeHint) { /* frame fully decoded */ 1850 if (zds->outEnd == zds->outStart) { /* output fully flushed */ 1851 if (zds->hostageByte) { 1852 if (input->pos >= input->size) { 1853 /* can't release hostage (not present) */ 1854 zds->streamStage = zdss_read; 1855 return 1; 1856 } 1857 input->pos++; /* release hostage */ 1858 } /* zds->hostageByte */ 1859 return 0; 1860 } /* zds->outEnd == zds->outStart */ 1861 if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ 1862 input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ 1863 zds->hostageByte=1; 1864 } 1865 return 1; 1866 } /* nextSrcSizeHint==0 */ 1867 nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ 1868 assert(zds->inPos <= nextSrcSizeHint); 1869 nextSrcSizeHint -= zds->inPos; /* part already loaded*/ 1870 return nextSrcSizeHint; 1871 } 1872 } 1873 1874 size_t ZSTD_decompressStream_simpleArgs ( 1875 ZSTD_DCtx* dctx, 1876 void* dst, size_t dstCapacity, size_t* dstPos, 1877 const void* src, size_t srcSize, size_t* srcPos) 1878 { 1879 ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; 1880 ZSTD_inBuffer input = { src, srcSize, *srcPos }; 1881 /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ 1882 size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); 1883 *dstPos = output.pos; 1884 *srcPos = input.pos; 1885 return cErr; 1886 } 1887