Lines Matching +full:assert +full:- +full:falling +full:- +full:edge
1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
6 * This source code is licensed under both the BSD-style license (found in the
9 * You may select, at your option, one of the above-listed licenses.
12 /*-*************************************
46 * Maximum size of the hash table dedicated to find 3-bytes matches,
57 /*-*************************************
62 * the one-pass compression functions.
75 /*-*************************************
90 … * row-based matchfinder. Unless the cdict is reloaded, we will use
102 assert(cctx != NULL); in ZSTD_initCCtx()
104 cctx->customMem = memManager; in ZSTD_initCCtx()
105 cctx->bmi2 = ZSTD_cpuSupportsBmi2(); in ZSTD_initCCtx()
107 assert(!ZSTD_isError(err)); in ZSTD_initCCtx()
115 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); in ZSTD_createCCtx_advanced()
129 if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ in ZSTD_initStaticCCtx()
136 ZSTD_cwksp_move(&cctx->workspace, &ws); in ZSTD_initStaticCCtx()
137 cctx->staticSize = workspaceSize; in ZSTD_initStaticCCtx()
140 …if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedB… in ZSTD_initStaticCCtx()
141 …cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->works… in ZSTD_initStaticCCtx()
142 …cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->works… in ZSTD_initStaticCCtx()
143 cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE); in ZSTD_initStaticCCtx()
144 cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; in ZSTD_initStaticCCtx()
145 cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); in ZSTD_initStaticCCtx()
154 ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); in ZSTD_clearAllDicts()
155 ZSTD_freeCDict(cctx->localDict.cdict); in ZSTD_clearAllDicts()
156 ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); in ZSTD_clearAllDicts()
157 ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); in ZSTD_clearAllDicts()
158 cctx->cdict = NULL; in ZSTD_clearAllDicts()
170 assert(cctx != NULL); in ZSTD_freeCCtxContent()
171 assert(cctx->staticSize == 0); in ZSTD_freeCCtxContent()
173 ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); in ZSTD_freeCCtxContent()
180 RETURN_ERROR_IF(cctx->staticSize, memory_allocation, in ZSTD_freeCCtx()
182 { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); in ZSTD_freeCCtx()
184 if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); in ZSTD_freeCCtx()
201 return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) in ZSTD_sizeof_CCtx()
202 + ZSTD_cwksp_sizeof(&cctx->workspace) in ZSTD_sizeof_CCtx()
203 + ZSTD_sizeof_localDict(cctx->localDict) in ZSTD_sizeof_CCtx()
213 const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } in ZSTD_getSeqStore()
224 assert(mode != ZSTD_ps_auto); in ZSTD_rowMatchFinderUsed()
237 if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; in ZSTD_resolveRowMatchFinderMode()
238 if (cParams->windowLog > kWindowLogLowerBound) mode = ZSTD_ps_enable; in ZSTD_resolveRowMatchFinderMode()
246 …return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_di… in ZSTD_resolveBlockSplitterMode()
253 assert(useRowMatchFinder != ZSTD_ps_auto); in ZSTD_allocateChainTable()
255 …* We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinde… in ZSTD_allocateChainTable()
267 …return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_di… in ZSTD_resolveEnableLdm()
295 return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; in ZSTD_CDictIndicesAreTagged()
310 assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); in ZSTD_makeCCtxParamsFromCParams()
311 assert(cctxParams.ldmParams.hashRateLog < 32); in ZSTD_makeCCtxParamsFromCParams()
319 assert(!ZSTD_checkCParams(cParams)); in ZSTD_makeCCtxParamsFromCParams()
332 params->customMem = customMem; in ZSTD_createCCtxParams_advanced()
344 ZSTD_customFree(params, params->customMem); in ZSTD_freeCCtxParams()
356 cctxParams->compressionLevel = compressionLevel; in ZSTD_CCtxParams_init()
357 cctxParams->fParams.contentSizeFlag = 1; in ZSTD_CCtxParams_init()
372 assert(!ZSTD_checkCParams(params->cParams)); in ZSTD_CCtxParams_init_internal()
374 cctxParams->cParams = params->cParams; in ZSTD_CCtxParams_init_internal()
375 cctxParams->fParams = params->fParams; in ZSTD_CCtxParams_init_internal()
379 cctxParams->compressionLevel = compressionLevel; in ZSTD_CCtxParams_init_internal()
380 …cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &par… in ZSTD_CCtxParams_init_internal()
381 …cctxParams->postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->postBlockSplitter, ¶… in ZSTD_CCtxParams_init_internal()
382 …cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->… in ZSTD_CCtxParams_init_internal()
383 …cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequenc… in ZSTD_CCtxParams_init_internal()
384 cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); in ZSTD_CCtxParams_init_internal()
385 …cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExt… in ZSTD_CCtxParams_init_internal()
387 … cctxParams->useRowMatchFinder, cctxParams->postBlockSplitter, cctxParams->ldmParams.enableLdm); in ZSTD_CCtxParams_init_internal()
405 assert(!ZSTD_checkCParams(params->cParams)); in ZSTD_CCtxParams_setZstdParams()
406 cctxParams->cParams = params->cParams; in ZSTD_CCtxParams_setZstdParams()
407 cctxParams->fParams = params->fParams; in ZSTD_CCtxParams_setZstdParams()
411 cctxParams->compressionLevel = ZSTD_NO_CLEVEL; in ZSTD_CCtxParams_setZstdParams()
693 if (cctx->streamStage != zcss_init) { in ZSTD_CCtx_setParameter()
695 cctx->cParamsChanged = 1; in ZSTD_CCtx_setParameter()
703 RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, in ZSTD_CCtx_setParameter()
749 return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); in ZSTD_CCtx_setParameter()
760 CCtxParams->format = (ZSTD_format_e)value; in ZSTD_CCtxParams_setParameter()
761 return (size_t)CCtxParams->format; in ZSTD_CCtxParams_setParameter()
766 CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ in ZSTD_CCtxParams_setParameter()
768 CCtxParams->compressionLevel = value; in ZSTD_CCtxParams_setParameter()
769 if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; in ZSTD_CCtxParams_setParameter()
776 CCtxParams->cParams.windowLog = (U32)value; in ZSTD_CCtxParams_setParameter()
777 return CCtxParams->cParams.windowLog; in ZSTD_CCtxParams_setParameter()
782 CCtxParams->cParams.hashLog = (U32)value; in ZSTD_CCtxParams_setParameter()
783 return CCtxParams->cParams.hashLog; in ZSTD_CCtxParams_setParameter()
788 CCtxParams->cParams.chainLog = (U32)value; in ZSTD_CCtxParams_setParameter()
789 return CCtxParams->cParams.chainLog; in ZSTD_CCtxParams_setParameter()
794 CCtxParams->cParams.searchLog = (U32)value; in ZSTD_CCtxParams_setParameter()
800 CCtxParams->cParams.minMatch = (U32)value; in ZSTD_CCtxParams_setParameter()
801 return CCtxParams->cParams.minMatch; in ZSTD_CCtxParams_setParameter()
805 CCtxParams->cParams.targetLength = (U32)value; in ZSTD_CCtxParams_setParameter()
806 return CCtxParams->cParams.targetLength; in ZSTD_CCtxParams_setParameter()
811 CCtxParams->cParams.strategy = (ZSTD_strategy)value; in ZSTD_CCtxParams_setParameter()
812 return (size_t)CCtxParams->cParams.strategy; in ZSTD_CCtxParams_setParameter()
817 CCtxParams->fParams.contentSizeFlag = value != 0; in ZSTD_CCtxParams_setParameter()
818 return (size_t)CCtxParams->fParams.contentSizeFlag; in ZSTD_CCtxParams_setParameter()
821 /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ in ZSTD_CCtxParams_setParameter()
822 CCtxParams->fParams.checksumFlag = value != 0; in ZSTD_CCtxParams_setParameter()
823 return (size_t)CCtxParams->fParams.checksumFlag; in ZSTD_CCtxParams_setParameter()
827 CCtxParams->fParams.noDictIDFlag = !value; in ZSTD_CCtxParams_setParameter()
828 return !CCtxParams->fParams.noDictIDFlag; in ZSTD_CCtxParams_setParameter()
831 CCtxParams->forceWindow = (value != 0); in ZSTD_CCtxParams_setParameter()
832 return (size_t)CCtxParams->forceWindow; in ZSTD_CCtxParams_setParameter()
837 CCtxParams->attachDictPref = pref; in ZSTD_CCtxParams_setParameter()
838 return CCtxParams->attachDictPref; in ZSTD_CCtxParams_setParameter()
844 CCtxParams->literalCompressionMode = lcm; in ZSTD_CCtxParams_setParameter()
845 return CCtxParams->literalCompressionMode; in ZSTD_CCtxParams_setParameter()
865 CCtxParams->enableDedicatedDictSearch = (value!=0); in ZSTD_CCtxParams_setParameter()
866 return (size_t)CCtxParams->enableDedicatedDictSearch; in ZSTD_CCtxParams_setParameter()
870 CCtxParams->ldmParams.enableLdm = (ZSTD_ParamSwitch_e)value; in ZSTD_CCtxParams_setParameter()
871 return CCtxParams->ldmParams.enableLdm; in ZSTD_CCtxParams_setParameter()
876 CCtxParams->ldmParams.hashLog = (U32)value; in ZSTD_CCtxParams_setParameter()
877 return CCtxParams->ldmParams.hashLog; in ZSTD_CCtxParams_setParameter()
882 CCtxParams->ldmParams.minMatchLength = (U32)value; in ZSTD_CCtxParams_setParameter()
883 return CCtxParams->ldmParams.minMatchLength; in ZSTD_CCtxParams_setParameter()
888 CCtxParams->ldmParams.bucketSizeLog = (U32)value; in ZSTD_CCtxParams_setParameter()
889 return CCtxParams->ldmParams.bucketSizeLog; in ZSTD_CCtxParams_setParameter()
894 CCtxParams->ldmParams.hashRateLog = (U32)value; in ZSTD_CCtxParams_setParameter()
895 return CCtxParams->ldmParams.hashRateLog; in ZSTD_CCtxParams_setParameter()
902 CCtxParams->targetCBlockSize = (U32)value; in ZSTD_CCtxParams_setParameter()
903 return CCtxParams->targetCBlockSize; in ZSTD_CCtxParams_setParameter()
908 CCtxParams->srcSizeHint = value; in ZSTD_CCtxParams_setParameter()
909 return (size_t)CCtxParams->srcSizeHint; in ZSTD_CCtxParams_setParameter()
913 CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; in ZSTD_CCtxParams_setParameter()
914 return CCtxParams->inBufferMode; in ZSTD_CCtxParams_setParameter()
918 CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; in ZSTD_CCtxParams_setParameter()
919 return CCtxParams->outBufferMode; in ZSTD_CCtxParams_setParameter()
923 CCtxParams->blockDelimiters = (ZSTD_SequenceFormat_e)value; in ZSTD_CCtxParams_setParameter()
924 return CCtxParams->blockDelimiters; in ZSTD_CCtxParams_setParameter()
928 CCtxParams->validateSequences = value; in ZSTD_CCtxParams_setParameter()
929 return (size_t)CCtxParams->validateSequences; in ZSTD_CCtxParams_setParameter()
933 CCtxParams->postBlockSplitter = (ZSTD_ParamSwitch_e)value; in ZSTD_CCtxParams_setParameter()
934 return CCtxParams->postBlockSplitter; in ZSTD_CCtxParams_setParameter()
938 CCtxParams->preBlockSplitter_level = value; in ZSTD_CCtxParams_setParameter()
939 return (size_t)CCtxParams->preBlockSplitter_level; in ZSTD_CCtxParams_setParameter()
943 CCtxParams->useRowMatchFinder = (ZSTD_ParamSwitch_e)value; in ZSTD_CCtxParams_setParameter()
944 return CCtxParams->useRowMatchFinder; in ZSTD_CCtxParams_setParameter()
948 CCtxParams->deterministicRefPrefix = !!value; in ZSTD_CCtxParams_setParameter()
949 return (size_t)CCtxParams->deterministicRefPrefix; in ZSTD_CCtxParams_setParameter()
953 CCtxParams->prefetchCDictTables = (ZSTD_ParamSwitch_e)value; in ZSTD_CCtxParams_setParameter()
954 return CCtxParams->prefetchCDictTables; in ZSTD_CCtxParams_setParameter()
958 CCtxParams->enableMatchFinderFallback = value; in ZSTD_CCtxParams_setParameter()
959 return (size_t)CCtxParams->enableMatchFinderFallback; in ZSTD_CCtxParams_setParameter()
964 assert(value>=0); in ZSTD_CCtxParams_setParameter()
965 CCtxParams->maxBlockSize = (size_t)value; in ZSTD_CCtxParams_setParameter()
966 return CCtxParams->maxBlockSize; in ZSTD_CCtxParams_setParameter()
970 CCtxParams->searchForExternalRepcodes = (ZSTD_ParamSwitch_e)value; in ZSTD_CCtxParams_setParameter()
971 return CCtxParams->searchForExternalRepcodes; in ZSTD_CCtxParams_setParameter()
979 return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); in ZSTD_CCtx_getParameter()
988 *value = (int)CCtxParams->format; in ZSTD_CCtxParams_getParameter()
991 *value = CCtxParams->compressionLevel; in ZSTD_CCtxParams_getParameter()
994 *value = (int)CCtxParams->cParams.windowLog; in ZSTD_CCtxParams_getParameter()
997 *value = (int)CCtxParams->cParams.hashLog; in ZSTD_CCtxParams_getParameter()
1000 *value = (int)CCtxParams->cParams.chainLog; in ZSTD_CCtxParams_getParameter()
1003 *value = (int)CCtxParams->cParams.searchLog; in ZSTD_CCtxParams_getParameter()
1006 *value = (int)CCtxParams->cParams.minMatch; in ZSTD_CCtxParams_getParameter()
1009 *value = (int)CCtxParams->cParams.targetLength; in ZSTD_CCtxParams_getParameter()
1012 *value = (int)CCtxParams->cParams.strategy; in ZSTD_CCtxParams_getParameter()
1015 *value = CCtxParams->fParams.contentSizeFlag; in ZSTD_CCtxParams_getParameter()
1018 *value = CCtxParams->fParams.checksumFlag; in ZSTD_CCtxParams_getParameter()
1021 *value = !CCtxParams->fParams.noDictIDFlag; in ZSTD_CCtxParams_getParameter()
1024 *value = CCtxParams->forceWindow; in ZSTD_CCtxParams_getParameter()
1027 *value = (int)CCtxParams->attachDictPref; in ZSTD_CCtxParams_getParameter()
1030 *value = (int)CCtxParams->literalCompressionMode; in ZSTD_CCtxParams_getParameter()
1033 assert(CCtxParams->nbWorkers == 0); in ZSTD_CCtxParams_getParameter()
1034 *value = CCtxParams->nbWorkers; in ZSTD_CCtxParams_getParameter()
1043 *value = CCtxParams->enableDedicatedDictSearch; in ZSTD_CCtxParams_getParameter()
1046 *value = (int)CCtxParams->ldmParams.enableLdm; in ZSTD_CCtxParams_getParameter()
1049 *value = (int)CCtxParams->ldmParams.hashLog; in ZSTD_CCtxParams_getParameter()
1052 *value = (int)CCtxParams->ldmParams.minMatchLength; in ZSTD_CCtxParams_getParameter()
1055 *value = (int)CCtxParams->ldmParams.bucketSizeLog; in ZSTD_CCtxParams_getParameter()
1058 *value = (int)CCtxParams->ldmParams.hashRateLog; in ZSTD_CCtxParams_getParameter()
1061 *value = (int)CCtxParams->targetCBlockSize; in ZSTD_CCtxParams_getParameter()
1064 *value = (int)CCtxParams->srcSizeHint; in ZSTD_CCtxParams_getParameter()
1067 *value = (int)CCtxParams->inBufferMode; in ZSTD_CCtxParams_getParameter()
1070 *value = (int)CCtxParams->outBufferMode; in ZSTD_CCtxParams_getParameter()
1073 *value = (int)CCtxParams->blockDelimiters; in ZSTD_CCtxParams_getParameter()
1076 *value = (int)CCtxParams->validateSequences; in ZSTD_CCtxParams_getParameter()
1079 *value = (int)CCtxParams->postBlockSplitter; in ZSTD_CCtxParams_getParameter()
1082 *value = CCtxParams->preBlockSplitter_level; in ZSTD_CCtxParams_getParameter()
1085 *value = (int)CCtxParams->useRowMatchFinder; in ZSTD_CCtxParams_getParameter()
1088 *value = (int)CCtxParams->deterministicRefPrefix; in ZSTD_CCtxParams_getParameter()
1091 *value = (int)CCtxParams->prefetchCDictTables; in ZSTD_CCtxParams_getParameter()
1094 *value = CCtxParams->enableMatchFinderFallback; in ZSTD_CCtxParams_getParameter()
1097 *value = (int)CCtxParams->maxBlockSize; in ZSTD_CCtxParams_getParameter()
1100 *value = (int)CCtxParams->searchForExternalRepcodes; in ZSTD_CCtxParams_getParameter()
1110 * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx.
1118 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_setParametersUsingCCtxParams()
1120 RETURN_ERROR_IF(cctx->cdict, stage_wrong, in ZSTD_CCtx_setParametersUsingCCtxParams()
1124 cctx->requestedParams = *params; in ZSTD_CCtx_setParametersUsingCCtxParams()
1169 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_setPledgedSrcSize()
1171 cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; in ZSTD_CCtx_setPledgedSrcSize()
1190 ZSTD_localDict* const dl = &cctx->localDict; in ZSTD_initLocalDict()
1191 if (dl->dict == NULL) { in ZSTD_initLocalDict()
1193 assert(dl->dictBuffer == NULL); in ZSTD_initLocalDict()
1194 assert(dl->cdict == NULL); in ZSTD_initLocalDict()
1195 assert(dl->dictSize == 0); in ZSTD_initLocalDict()
1198 if (dl->cdict != NULL) { in ZSTD_initLocalDict()
1200 assert(cctx->cdict == dl->cdict); in ZSTD_initLocalDict()
1203 assert(dl->dictSize > 0); in ZSTD_initLocalDict()
1204 assert(cctx->cdict == NULL); in ZSTD_initLocalDict()
1205 assert(cctx->prefixDict.dict == NULL); in ZSTD_initLocalDict()
1207 dl->cdict = ZSTD_createCDict_advanced2( in ZSTD_initLocalDict()
1208 dl->dict, in ZSTD_initLocalDict()
1209 dl->dictSize, in ZSTD_initLocalDict()
1211 dl->dictContentType, in ZSTD_initLocalDict()
1212 &cctx->requestedParams, in ZSTD_initLocalDict()
1213 cctx->customMem); in ZSTD_initLocalDict()
1214 RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); in ZSTD_initLocalDict()
1215 cctx->cdict = dl->cdict; in ZSTD_initLocalDict()
1226 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_loadDictionary_advanced()
1232 cctx->localDict.dict = dict; in ZSTD_CCtx_loadDictionary_advanced()
1236 RETURN_ERROR_IF(cctx->staticSize, memory_allocation, in ZSTD_CCtx_loadDictionary_advanced()
1238 dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); in ZSTD_CCtx_loadDictionary_advanced()
1242 cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ in ZSTD_CCtx_loadDictionary_advanced()
1243 cctx->localDict.dict = dictBuffer; /* read-only reference */ in ZSTD_CCtx_loadDictionary_advanced()
1245 cctx->localDict.dictSize = dictSize; in ZSTD_CCtx_loadDictionary_advanced()
1246 cctx->localDict.dictContentType = dictContentType; in ZSTD_CCtx_loadDictionary_advanced()
1266 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_refCDict()
1270 cctx->cdict = cdict; in ZSTD_CCtx_refCDict()
1276 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_refThreadPool()
1278 cctx->pool = pool; in ZSTD_CCtx_refThreadPool()
1290 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_refPrefix_advanced()
1294 cctx->prefixDict.dict = prefix; in ZSTD_CCtx_refPrefix_advanced()
1295 cctx->prefixDict.dictSize = prefixSize; in ZSTD_CCtx_refPrefix_advanced()
1296 cctx->prefixDict.dictContentType = dictContentType; in ZSTD_CCtx_refPrefix_advanced()
1307 cctx->streamStage = zcss_init; in ZSTD_CCtx_reset()
1308 cctx->pledgedSrcSizePlusOne = 0; in ZSTD_CCtx_reset()
1312 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, in ZSTD_CCtx_reset()
1315 return ZSTD_CCtxParams_reset(&cctx->requestedParams); in ZSTD_CCtx_reset()
1364 return hashLog - btScale; in ZSTD_cycleLog()
1382 assert(windowLog <= ZSTD_WINDOWLOG_MAX); in ZSTD_dictAndWindowLog()
1383 assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ in ZSTD_dictAndWindowLog()
1396 return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; in ZSTD_dictAndWindowLog()
1416 const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); in ZSTD_adjustCParams_internal()
1417 assert(ZSTD_checkCParams(cPar)==0); in ZSTD_adjustCParams_internal()
1419 /* Cascade the selected strategy down to the next-highest one built into in ZSTD_adjustCParams_internal()
1484 assert(0); in ZSTD_adjustCParams_internal()
1494 ZSTD_highbit32(tSize-1) + 1; in ZSTD_adjustCParams_internal()
1502 cPar.chainLog -= (cycleLog - dictAndWindowLog); in ZSTD_adjustCParams_internal()
1512 U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; in ZSTD_adjustCParams_internal()
1530 /* We can't hash more than 32-bits in total. So that means that we require: in ZSTD_adjustCParams_internal()
1531 * (hashLog - rowLog + 8) <= 32 in ZSTD_adjustCParams_internal()
1534 /* Switch to 32-entry rows if searchLog is 5 (or more) */ in ZSTD_adjustCParams_internal()
1536 U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; in ZSTD_adjustCParams_internal()
1538 assert(cPar.hashLog >= rowLog); in ZSTD_adjustCParams_internal()
1564 if (overrides->windowLog) cParams->windowLog = overrides->windowLog; in ZSTD_overrideCParams()
1565 if (overrides->hashLog) cParams->hashLog = overrides->hashLog; in ZSTD_overrideCParams()
1566 if (overrides->chainLog) cParams->chainLog = overrides->chainLog; in ZSTD_overrideCParams()
1567 if (overrides->searchLog) cParams->searchLog = overrides->searchLog; in ZSTD_overrideCParams()
1568 if (overrides->minMatch) cParams->minMatch = overrides->minMatch; in ZSTD_overrideCParams()
1569 if (overrides->targetLength) cParams->targetLength = overrides->targetLength; in ZSTD_overrideCParams()
1570 if (overrides->strategy) cParams->strategy = overrides->strategy; in ZSTD_overrideCParams()
1577 if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { in ZSTD_getCParamsFromCCtxParams()
1578 assert(CCtxParams->srcSizeHint>=0); in ZSTD_getCParamsFromCCtxParams()
1579 srcSizeHint = (U64)CCtxParams->srcSizeHint; in ZSTD_getCParamsFromCCtxParams()
1581 cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); in ZSTD_getCParamsFromCCtxParams()
1582 …if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW… in ZSTD_getCParamsFromCCtxParams()
1583 ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); in ZSTD_getCParamsFromCCtxParams()
1584 assert(!ZSTD_checkCParams(cParams)); in ZSTD_getCParamsFromCCtxParams()
1586 …return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFi… in ZSTD_getCParamsFromCCtxParams()
1595 /* chain table size should be 0 for fast or row-hash strategies */ in ZSTD_sizeof_matchState()
1596 …size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedic… in ZSTD_sizeof_matchState()
1597 ? ((size_t)1 << cParams->chainLog) in ZSTD_sizeof_matchState()
1599 size_t const hSize = ((size_t)1) << cParams->hashLog; in ZSTD_sizeof_matchState()
1600 …U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->window… in ZSTD_sizeof_matchState()
1614 size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder) in ZSTD_sizeof_matchState()
1617 size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) in ZSTD_sizeof_matchState()
1624 assert(useRowMatchFinder != ZSTD_ps_auto); in ZSTD_sizeof_matchState()
1626 DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", in ZSTD_sizeof_matchState()
1649 size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); in ZSTD_estimateCCtxSize_usingCCtxParams_internal()
1651 size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); in ZSTD_estimateCCtxSize_usingCCtxParams_internal()
1661 size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? in ZSTD_estimateCCtxSize_usingCCtxParams_internal()
1694 …ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFin… in ZSTD_estimateCCtxSize_usingCCtxParams()
1697 …RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-thread… in ZSTD_estimateCCtxSize_usingCCtxParams()
1698 /* estimateCCtxSize is for one-shot compression. So no buffers should in ZSTD_estimateCCtxSize_usingCCtxParams()
1699 * be needed. However, we still allocate two 0-sized buffers, which can in ZSTD_estimateCCtxSize_usingCCtxParams()
1702 …&cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeq… in ZSTD_estimateCCtxSize_usingCCtxParams()
1709 … /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ in ZSTD_estimateCCtxSize_usingCParams()
1749 …RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-thread… in ZSTD_estimateCStreamSize_usingCCtxParams()
1752 …size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.… in ZSTD_estimateCStreamSize_usingCCtxParams()
1753 size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) in ZSTD_estimateCStreamSize_usingCCtxParams()
1756 size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) in ZSTD_estimateCStreamSize_usingCCtxParams()
1759 …onst useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParam… in ZSTD_estimateCStreamSize_usingCCtxParams()
1762 &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, in ZSTD_estimateCStreamSize_usingCCtxParams()
1763 ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); in ZSTD_estimateCStreamSize_usingCCtxParams()
1771 … /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ in ZSTD_estimateCStreamSize_usingCParams()
1803 * able to count progression inside worker threads (non-blocking mode).
1808 size_t const buffered = (cctx->inBuff == NULL) ? 0 : in ZSTD_getFrameProgression()
1809 cctx->inBuffPos - cctx->inToCompress; in ZSTD_getFrameProgression()
1810 if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); in ZSTD_getFrameProgression()
1811 assert(buffered <= ZSTD_BLOCKSIZE_MAX); in ZSTD_getFrameProgression()
1812 fp.ingested = cctx->consumedSrcSize + buffered; in ZSTD_getFrameProgression()
1813 fp.consumed = cctx->consumedSrcSize; in ZSTD_getFrameProgression()
1814 fp.produced = cctx->producedCSize; in ZSTD_getFrameProgression()
1815 …fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming… in ZSTD_getFrameProgression()
1827 …return 0; /* over-simplification; could also check if context is currently running in streaming … in ZSTD_toFlushNow()
1835 assert(cParams1.windowLog == cParams2.windowLog); in ZSTD_assertEqualCParams()
1836 assert(cParams1.chainLog == cParams2.chainLog); in ZSTD_assertEqualCParams()
1837 assert(cParams1.hashLog == cParams2.hashLog); in ZSTD_assertEqualCParams()
1838 assert(cParams1.searchLog == cParams2.searchLog); in ZSTD_assertEqualCParams()
1839 assert(cParams1.minMatch == cParams2.minMatch); in ZSTD_assertEqualCParams()
1840 assert(cParams1.targetLength == cParams2.targetLength); in ZSTD_assertEqualCParams()
1841 assert(cParams1.strategy == cParams2.strategy); in ZSTD_assertEqualCParams()
1848 bs->rep[i] = repStartValue[i]; in ZSTD_reset_compressedBlockState()
1849 bs->entropy.huf.repeatMode = HUF_repeat_none; in ZSTD_reset_compressedBlockState()
1850 bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; in ZSTD_reset_compressedBlockState()
1851 bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; in ZSTD_reset_compressedBlockState()
1852 bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; in ZSTD_reset_compressedBlockState()
1861 ZSTD_window_clear(&ms->window); in ZSTD_invalidateMatchState()
1863 ms->nextToUpdate = ms->window.dictLimit; in ZSTD_invalidateMatchState()
1864 ms->loadedDictEnd = 0; in ZSTD_invalidateMatchState()
1865 ms->opt.litLengthSum = 0; /* force reset of btopt stats */ in ZSTD_invalidateMatchState()
1866 ms->dictMatchState = NULL; in ZSTD_invalidateMatchState()
1907 ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); in ZSTD_advanceHashSalt()
1919 /* disable chain table allocation for fast or row-based strategies */ in ZSTD_reset_matchState()
1920 size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, in ZSTD_reset_matchState()
1921 … ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) in ZSTD_reset_matchState()
1922 ? ((size_t)1 << cParams->chainLog) in ZSTD_reset_matchState()
1924 size_t const hSize = ((size_t)1) << cParams->hashLog; in ZSTD_reset_matchState()
1925 …Log3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParam… in ZSTD_reset_matchState()
1929 assert(useRowMatchFinder != ZSTD_ps_auto); in ZSTD_reset_matchState()
1931 ZSTD_window_init(&ms->window); in ZSTD_reset_matchState()
1935 ms->hashLog3 = hashLog3; in ZSTD_reset_matchState()
1936 ms->lazySkipping = 0; in ZSTD_reset_matchState()
1940 assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ in ZSTD_reset_matchState()
1946 ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); in ZSTD_reset_matchState()
1947 ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); in ZSTD_reset_matchState()
1948 ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); in ZSTD_reset_matchState()
1958 if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { in ZSTD_reset_matchState()
1964 ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); in ZSTD_reset_matchState()
1968 ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize); in ZSTD_reset_matchState()
1969 ZSTD_memset(ms->tagTable, 0, tagTableSize); in ZSTD_reset_matchState()
1970 ms->hashSalt = 0; in ZSTD_reset_matchState()
1972 { /* Switch to 32-entry rows if searchLog is 5 (or more) */ in ZSTD_reset_matchState()
1973 U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); in ZSTD_reset_matchState()
1974 assert(cParams->hashLog >= rowLog); in ZSTD_reset_matchState()
1975 ms->rowHashLog = cParams->hashLog - rowLog; in ZSTD_reset_matchState()
1980 if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { in ZSTD_reset_matchState()
1982 … ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<<Litbits) * sizeof(unsigned)); in ZSTD_reset_matchState()
1983 … ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned)); in ZSTD_reset_matchState()
1984 …ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned)… in ZSTD_reset_matchState()
1985 … ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned)); in ZSTD_reset_matchState()
1986 …ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_m… in ZSTD_reset_matchState()
1987 …ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD… in ZSTD_reset_matchState()
1990 ms->cParams = *cParams; in ZSTD_reset_matchState()
2000 * Works when re-using a context for a lot of smallish inputs :
2007 return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); in ZSTD_indexTooCloseToMax()
2033 ZSTD_cwksp* const ws = &zc->workspace; in ZSTD_resetCCtx_internal()
2035 …(U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->postB… in ZSTD_resetCCtx_internal()
2036 assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); in ZSTD_resetCCtx_internal()
2038 zc->isFirstBlock = 1; in ZSTD_resetCCtx_internal()
2043 zc->appliedParams = *params; in ZSTD_resetCCtx_internal()
2044 params = &zc->appliedParams; in ZSTD_resetCCtx_internal()
2046 assert(params->useRowMatchFinder != ZSTD_ps_auto); in ZSTD_resetCCtx_internal()
2047 assert(params->postBlockSplitter != ZSTD_ps_auto); in ZSTD_resetCCtx_internal()
2048 assert(params->ldmParams.enableLdm != ZSTD_ps_auto); in ZSTD_resetCCtx_internal()
2049 assert(params->maxBlockSize != 0); in ZSTD_resetCCtx_internal()
2050 if (params->ldmParams.enableLdm == ZSTD_ps_enable) { in ZSTD_resetCCtx_internal()
2052 ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); in ZSTD_resetCCtx_internal()
2053 assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); in ZSTD_resetCCtx_internal()
2054 assert(params->ldmParams.hashRateLog < 32); in ZSTD_resetCCtx_internal()
2057 …{ size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrc… in ZSTD_resetCCtx_internal()
2058 size_t const blockSize = MIN(params->maxBlockSize, windowSize); in ZSTD_resetCCtx_internal()
2059 …size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, ZSTD_hasExtSeqProd(para… in ZSTD_resetCCtx_internal()
2060 … size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) in ZSTD_resetCCtx_internal()
2063 … size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) in ZSTD_resetCCtx_internal()
2066 size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); in ZSTD_resetCCtx_internal()
2068 int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); in ZSTD_resetCCtx_internal()
2071 (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; in ZSTD_resetCCtx_internal()
2075 … ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, in ZSTD_resetCCtx_internal()
2076 … buffInSize, buffOutSize, pledgedSrcSize, ZSTD_hasExtSeqProd(params), params->maxBlockSize); in ZSTD_resetCCtx_internal()
2080 if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); in ZSTD_resetCCtx_internal()
2087 DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); in ZSTD_resetCCtx_internal()
2094 RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); in ZSTD_resetCCtx_internal()
2098 ZSTD_cwksp_free(ws, zc->customMem); in ZSTD_resetCCtx_internal()
2099 FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); in ZSTD_resetCCtx_internal()
2105 assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); in ZSTD_resetCCtx_internal()
2106 …zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZS… in ZSTD_resetCCtx_internal()
2107 …RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBloc… in ZSTD_resetCCtx_internal()
2108 …zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZS… in ZSTD_resetCCtx_internal()
2109 …RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBloc… in ZSTD_resetCCtx_internal()
2110 zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE); in ZSTD_resetCCtx_internal()
2111 … RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace"); in ZSTD_resetCCtx_internal()
2112 zc->tmpWkspSize = TMP_WORKSPACE_SIZE; in ZSTD_resetCCtx_internal()
2118 zc->blockState.matchState.cParams = params->cParams; in ZSTD_resetCCtx_internal()
2119 … zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; in ZSTD_resetCCtx_internal()
2120 zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; in ZSTD_resetCCtx_internal()
2121 zc->consumedSrcSize = 0; in ZSTD_resetCCtx_internal()
2122 zc->producedCSize = 0; in ZSTD_resetCCtx_internal()
2124 zc->appliedParams.fParams.contentSizeFlag = 0; in ZSTD_resetCCtx_internal()
2126 (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); in ZSTD_resetCCtx_internal()
2127 zc->blockSizeMax = blockSize; in ZSTD_resetCCtx_internal()
2129 xxh64_reset(&zc->xxhState, 0); in ZSTD_resetCCtx_internal()
2130 zc->stage = ZSTDcs_init; in ZSTD_resetCCtx_internal()
2131 zc->dictID = 0; in ZSTD_resetCCtx_internal()
2132 zc->dictContentSize = 0; in ZSTD_resetCCtx_internal()
2134 ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); in ZSTD_resetCCtx_internal()
2137 &zc->blockState.matchState, in ZSTD_resetCCtx_internal()
2139 ¶ms->cParams, in ZSTD_resetCCtx_internal()
2140 params->useRowMatchFinder, in ZSTD_resetCCtx_internal()
2145 …zc->seqStore.sequencesStart = (SeqDef*)ZSTD_cwksp_reserve_aligned64(ws, maxNbSeq * sizeof(SeqDef)); in ZSTD_resetCCtx_internal()
2148 if (params->ldmParams.enableLdm == ZSTD_ps_enable) { in ZSTD_resetCCtx_internal()
2150 size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; in ZSTD_resetCCtx_internal()
2151 …zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned64(ws, ldmHSize * sizeof(ldmEntry_… in ZSTD_resetCCtx_internal()
2152 ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); in ZSTD_resetCCtx_internal()
2153 … zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned64(ws, maxNbLdmSeq * sizeof(rawSeq)); in ZSTD_resetCCtx_internal()
2154 zc->maxNbLdmSequences = maxNbLdmSeq; in ZSTD_resetCCtx_internal()
2156 ZSTD_window_init(&zc->ldmState.window); in ZSTD_resetCCtx_internal()
2157 zc->ldmState.loadedDictEnd = 0; in ZSTD_resetCCtx_internal()
2160 /* reserve space for block-level external sequences */ in ZSTD_resetCCtx_internal()
2163 zc->extSeqBufCapacity = maxNbExternalSeq; in ZSTD_resetCCtx_internal()
2164 zc->extSeqBuf = in ZSTD_resetCCtx_internal()
2173 zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); in ZSTD_resetCCtx_internal()
2174 zc->seqStore.maxNbLit = blockSize; in ZSTD_resetCCtx_internal()
2176 zc->bufferedPolicy = zbuff; in ZSTD_resetCCtx_internal()
2177 zc->inBuffSize = buffInSize; in ZSTD_resetCCtx_internal()
2178 zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); in ZSTD_resetCCtx_internal()
2179 zc->outBuffSize = buffOutSize; in ZSTD_resetCCtx_internal()
2180 zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); in ZSTD_resetCCtx_internal()
2183 if (params->ldmParams.enableLdm == ZSTD_ps_enable) { in ZSTD_resetCCtx_internal()
2186 ((size_t)1) << (params->ldmParams.hashLog - in ZSTD_resetCCtx_internal()
2187 params->ldmParams.bucketSizeLog); in ZSTD_resetCCtx_internal()
2188 zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); in ZSTD_resetCCtx_internal()
2189 ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); in ZSTD_resetCCtx_internal()
2194 zc->seqStore.maxNbSeq = maxNbSeq; in ZSTD_resetCCtx_internal()
2195 zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); in ZSTD_resetCCtx_internal()
2196 zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); in ZSTD_resetCCtx_internal()
2197 zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); in ZSTD_resetCCtx_internal()
2200 assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); in ZSTD_resetCCtx_internal()
2202 zc->initialized = 1; in ZSTD_resetCCtx_internal()
2214 for (i=0; i<ZSTD_REP_NUM; i++) cctx->blockState.prevCBlock->rep[i] = 0; in ZSTD_invalidateRepCodes()
2215 assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); in ZSTD_invalidateRepCodes()
2220 * in-place.
2239 size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; in ZSTD_shouldAttachDict()
2240 int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; in ZSTD_shouldAttachDict()
2244 || params->attachDictPref == ZSTD_dictForceAttach ) in ZSTD_shouldAttachDict()
2245 && params->attachDictPref != ZSTD_dictForceCopy in ZSTD_shouldAttachDict()
2246 && !params->forceWindow ); /* dictMatchState isn't correctly in ZSTD_shouldAttachDict()
2260 ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; in ZSTD_resetCCtx_byAttachingCDict()
2262 assert(windowLog != 0); in ZSTD_resetCCtx_byAttachingCDict()
2267 if (cdict->matchState.dedicatedDictSearch) { in ZSTD_resetCCtx_byAttachingCDict()
2272 cdict->dictContentSize, ZSTD_cpm_attachDict, in ZSTD_resetCCtx_byAttachingCDict()
2275 params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ in ZSTD_resetCCtx_byAttachingCDict()
2279 assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); in ZSTD_resetCCtx_byAttachingCDict()
2282 { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc in ZSTD_resetCCtx_byAttachingCDict()
2283 - cdict->matchState.window.base); in ZSTD_resetCCtx_byAttachingCDict()
2284 const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; in ZSTD_resetCCtx_byAttachingCDict()
2290 cctx->blockState.matchState.dictMatchState = &cdict->matchState; in ZSTD_resetCCtx_byAttachingCDict()
2294 if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { in ZSTD_resetCCtx_byAttachingCDict()
2295 cctx->blockState.matchState.window.nextSrc = in ZSTD_resetCCtx_byAttachingCDict()
2296 cctx->blockState.matchState.window.base + cdictEnd; in ZSTD_resetCCtx_byAttachingCDict()
2297 ZSTD_window_clear(&cctx->blockState.matchState.window); in ZSTD_resetCCtx_byAttachingCDict()
2300 … cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; in ZSTD_resetCCtx_byAttachingCDict()
2303 cctx->dictID = cdict->dictID; in ZSTD_resetCCtx_byAttachingCDict()
2304 cctx->dictContentSize = cdict->dictContentSize; in ZSTD_resetCCtx_byAttachingCDict()
2307 ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); in ZSTD_resetCCtx_byAttachingCDict()
2334 const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; in ZSTD_resetCCtx_byCopyingCDict()
2336 assert(!cdict->matchState.dedicatedDictSearch); in ZSTD_resetCCtx_byCopyingCDict()
2341 assert(windowLog != 0); in ZSTD_resetCCtx_byCopyingCDict()
2345 params.useRowMatchFinder = cdict->useRowMatchFinder; in ZSTD_resetCCtx_byCopyingCDict()
2349 assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); in ZSTD_resetCCtx_byCopyingCDict()
2350 assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); in ZSTD_resetCCtx_byCopyingCDict()
2351 assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); in ZSTD_resetCCtx_byCopyingCDict()
2354 ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); in ZSTD_resetCCtx_byCopyingCDict()
2355 assert(params.useRowMatchFinder != ZSTD_ps_auto); in ZSTD_resetCCtx_byCopyingCDict()
2358 …{ size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFi… in ZSTD_resetCCtx_byCopyingCDict()
2359 ? ((size_t)1 << cdict_cParams->chainLog) in ZSTD_resetCCtx_byCopyingCDict()
2361 size_t const hSize = (size_t)1 << cdict_cParams->hashLog; in ZSTD_resetCCtx_byCopyingCDict()
2363 ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, in ZSTD_resetCCtx_byCopyingCDict()
2364 cdict->matchState.hashTable, in ZSTD_resetCCtx_byCopyingCDict()
2368 …if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchF… in ZSTD_resetCCtx_byCopyingCDict()
2369 ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, in ZSTD_resetCCtx_byCopyingCDict()
2370 cdict->matchState.chainTable, in ZSTD_resetCCtx_byCopyingCDict()
2374 if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { in ZSTD_resetCCtx_byCopyingCDict()
2376 ZSTD_memcpy(cctx->blockState.matchState.tagTable, in ZSTD_resetCCtx_byCopyingCDict()
2377 cdict->matchState.tagTable, in ZSTD_resetCCtx_byCopyingCDict()
2379 cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; in ZSTD_resetCCtx_byCopyingCDict()
2384 assert(cctx->blockState.matchState.hashLog3 <= 31); in ZSTD_resetCCtx_byCopyingCDict()
2385 { U32 const h3log = cctx->blockState.matchState.hashLog3; in ZSTD_resetCCtx_byCopyingCDict()
2387 assert(cdict->matchState.hashLog3 == 0); in ZSTD_resetCCtx_byCopyingCDict()
2388 ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); in ZSTD_resetCCtx_byCopyingCDict()
2391 ZSTD_cwksp_mark_tables_clean(&cctx->workspace); in ZSTD_resetCCtx_byCopyingCDict()
2394 { ZSTD_MatchState_t const* srcMatchState = &cdict->matchState; in ZSTD_resetCCtx_byCopyingCDict()
2395 ZSTD_MatchState_t* dstMatchState = &cctx->blockState.matchState; in ZSTD_resetCCtx_byCopyingCDict()
2396 dstMatchState->window = srcMatchState->window; in ZSTD_resetCCtx_byCopyingCDict()
2397 dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; in ZSTD_resetCCtx_byCopyingCDict()
2398 dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; in ZSTD_resetCCtx_byCopyingCDict()
2401 cctx->dictID = cdict->dictID; in ZSTD_resetCCtx_byCopyingCDict()
2402 cctx->dictContentSize = cdict->dictContentSize; in ZSTD_resetCCtx_byCopyingCDict()
2405 ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); in ZSTD_resetCCtx_byCopyingCDict()
2412 * in-place. We decide here which strategy to use. */
2445 RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, in ZSTD_copyCCtx_internal()
2448 ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); in ZSTD_copyCCtx_internal()
2449 { ZSTD_CCtx_params params = dstCCtx->requestedParams; in ZSTD_copyCCtx_internal()
2451 params.cParams = srcCCtx->appliedParams.cParams; in ZSTD_copyCCtx_internal()
2452 assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); in ZSTD_copyCCtx_internal()
2453 assert(srcCCtx->appliedParams.postBlockSplitter != ZSTD_ps_auto); in ZSTD_copyCCtx_internal()
2454 assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); in ZSTD_copyCCtx_internal()
2455 params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; in ZSTD_copyCCtx_internal()
2456 params.postBlockSplitter = srcCCtx->appliedParams.postBlockSplitter; in ZSTD_copyCCtx_internal()
2457 params.ldmParams = srcCCtx->appliedParams.ldmParams; in ZSTD_copyCCtx_internal()
2459 params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; in ZSTD_copyCCtx_internal()
2463 … assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); in ZSTD_copyCCtx_internal()
2464 assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); in ZSTD_copyCCtx_internal()
2465 assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); in ZSTD_copyCCtx_internal()
2466 assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); in ZSTD_copyCCtx_internal()
2467 assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); in ZSTD_copyCCtx_internal()
2470 ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); in ZSTD_copyCCtx_internal()
2473 { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, in ZSTD_copyCCtx_internal()
2474 srcCCtx->appliedParams.useRowMatchFinder, in ZSTD_copyCCtx_internal()
2476 ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) in ZSTD_copyCCtx_internal()
2478 size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; in ZSTD_copyCCtx_internal()
2479 U32 const h3log = srcCCtx->blockState.matchState.hashLog3; in ZSTD_copyCCtx_internal()
2482 ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, in ZSTD_copyCCtx_internal()
2483 srcCCtx->blockState.matchState.hashTable, in ZSTD_copyCCtx_internal()
2485 ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, in ZSTD_copyCCtx_internal()
2486 srcCCtx->blockState.matchState.chainTable, in ZSTD_copyCCtx_internal()
2488 ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, in ZSTD_copyCCtx_internal()
2489 srcCCtx->blockState.matchState.hashTable3, in ZSTD_copyCCtx_internal()
2493 ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); in ZSTD_copyCCtx_internal()
2497 const ZSTD_MatchState_t* srcMatchState = &srcCCtx->blockState.matchState; in ZSTD_copyCCtx_internal()
2498 ZSTD_MatchState_t* dstMatchState = &dstCCtx->blockState.matchState; in ZSTD_copyCCtx_internal()
2499 dstMatchState->window = srcMatchState->window; in ZSTD_copyCCtx_internal()
2500 dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; in ZSTD_copyCCtx_internal()
2501 dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; in ZSTD_copyCCtx_internal()
2503 dstCCtx->dictID = srcCCtx->dictID; in ZSTD_copyCCtx_internal()
2504 dstCCtx->dictContentSize = srcCCtx->dictContentSize; in ZSTD_copyCCtx_internal()
2507 …ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->block… in ZSTD_copyCCtx_internal()
2520 ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; in ZSTD_copyCCtx()
2537 * to help auto-vectorization */
2546 assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ in ZSTD_reduceTable_internal()
2547 assert(size < (1U<<31)); /* can be cast to int */ in ZSTD_reduceTable_internal()
2556 * to auto-vectorize the loop. */ in ZSTD_reduceTable_internal()
2561 newVal = table[cellNb] - reducerValue; in ZSTD_reduceTable_internal()
2582 { U32 const hSize = (U32)1 << params->cParams.hashLog; in ZSTD_reduceIndex()
2583 ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); in ZSTD_reduceIndex()
2586 …if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicate… in ZSTD_reduceIndex()
2587 U32 const chainSize = (U32)1 << params->cParams.chainLog; in ZSTD_reduceIndex()
2588 if (params->cParams.strategy == ZSTD_btlazy2) in ZSTD_reduceIndex()
2589 ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); in ZSTD_reduceIndex()
2591 ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); in ZSTD_reduceIndex()
2594 if (ms->hashLog3) { in ZSTD_reduceIndex()
2595 U32 const h3Size = (U32)1 << ms->hashLog3; in ZSTD_reduceIndex()
2596 ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); in ZSTD_reduceIndex()
2601 /*-*******************************************************
2609 const SeqDef* const sequences = seqStorePtr->sequencesStart; in ZSTD_seqToCodes()
2610 BYTE* const llCodeTable = seqStorePtr->llCode; in ZSTD_seqToCodes()
2611 BYTE* const ofCodeTable = seqStorePtr->ofCode; in ZSTD_seqToCodes()
2612 BYTE* const mlCodeTable = seqStorePtr->mlCode; in ZSTD_seqToCodes()
2613 U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); in ZSTD_seqToCodes()
2616 assert(nbSeq <= seqStorePtr->maxNbSeq); in ZSTD_seqToCodes()
2624 assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN)); in ZSTD_seqToCodes()
2628 if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) in ZSTD_seqToCodes()
2629 llCodeTable[seqStorePtr->longLengthPos] = MaxLL; in ZSTD_seqToCodes()
2630 if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) in ZSTD_seqToCodes()
2631 mlCodeTable[seqStorePtr->longLengthPos] = MaxML; in ZSTD_seqToCodes()
2641 DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); in ZSTD_useTargetCBlockSize()
2642 return (cctxParams->targetCBlockSize != 0); in ZSTD_useTargetCBlockSize()
2652 DEBUGLOG(5, "ZSTD_blockSplitterEnabled (postBlockSplitter=%d)", cctxParams->postBlockSplitter); in ZSTD_blockSplitterEnabled()
2653 assert(cctxParams->postBlockSplitter != ZSTD_ps_auto); in ZSTD_blockSplitterEnabled()
2654 return (cctxParams->postBlockSplitter == ZSTD_ps_enable); in ZSTD_blockSplitterEnabled()
2674 * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
2687 FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; in ZSTD_buildSequencesStatistics()
2688 FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; in ZSTD_buildSequencesStatistics()
2689 FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; in ZSTD_buildSequencesStatistics()
2690 const BYTE* const ofCodeTable = seqStorePtr->ofCode; in ZSTD_buildSequencesStatistics()
2691 const BYTE* const llCodeTable = seqStorePtr->llCode; in ZSTD_buildSequencesStatistics()
2692 const BYTE* const mlCodeTable = seqStorePtr->mlCode; in ZSTD_buildSequencesStatistics()
2698 assert(op <= oend); in ZSTD_buildSequencesStatistics()
2699 assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ in ZSTD_buildSequencesStatistics()
2704 nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; in ZSTD_buildSequencesStatistics()
2705 stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, in ZSTD_buildSequencesStatistics()
2707 LLFSELog, prevEntropy->litlengthCTable, in ZSTD_buildSequencesStatistics()
2710 assert(set_basic < set_compressed && set_rle < set_compressed); in ZSTD_buildSequencesStatistics()
2711 …assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); … in ZSTD_buildSequencesStatistics()
2713 op, (size_t)(oend - op), in ZSTD_buildSequencesStatistics()
2717 prevEntropy->litlengthCTable, in ZSTD_buildSequencesStatistics()
2718 sizeof(prevEntropy->litlengthCTable), in ZSTD_buildSequencesStatistics()
2728 assert(op <= oend); in ZSTD_buildSequencesStatistics()
2737 nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; in ZSTD_buildSequencesStatistics()
2738 stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, in ZSTD_buildSequencesStatistics()
2740 OffFSELog, prevEntropy->offcodeCTable, in ZSTD_buildSequencesStatistics()
2743 …assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /… in ZSTD_buildSequencesStatistics()
2745 op, (size_t)(oend - op), in ZSTD_buildSequencesStatistics()
2749 prevEntropy->offcodeCTable, in ZSTD_buildSequencesStatistics()
2750 sizeof(prevEntropy->offcodeCTable), in ZSTD_buildSequencesStatistics()
2760 assert(op <= oend); in ZSTD_buildSequencesStatistics()
2766 DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); in ZSTD_buildSequencesStatistics()
2767 nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; in ZSTD_buildSequencesStatistics()
2768 stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, in ZSTD_buildSequencesStatistics()
2770 MLFSELog, prevEntropy->matchlengthCTable, in ZSTD_buildSequencesStatistics()
2773 …assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none))… in ZSTD_buildSequencesStatistics()
2775 op, (size_t)(oend - op), in ZSTD_buildSequencesStatistics()
2779 prevEntropy->matchlengthCTable, in ZSTD_buildSequencesStatistics()
2780 sizeof(prevEntropy->matchlengthCTable), in ZSTD_buildSequencesStatistics()
2790 assert(op <= oend); in ZSTD_buildSequencesStatistics()
2792 stats.size = (size_t)(op-ostart); in ZSTD_buildSequencesStatistics()
2812 ZSTD_strategy const strategy = cctxParams->cParams.strategy; in ZSTD_entropyCompressSeqStore_internal()
2814 FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; in ZSTD_entropyCompressSeqStore_internal()
2815 FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; in ZSTD_entropyCompressSeqStore_internal()
2816 FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; in ZSTD_entropyCompressSeqStore_internal()
2817 const SeqDef* const sequences = seqStorePtr->sequencesStart; in ZSTD_entropyCompressSeqStore_internal()
2818 const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); in ZSTD_entropyCompressSeqStore_internal()
2819 const BYTE* const ofCodeTable = seqStorePtr->ofCode; in ZSTD_entropyCompressSeqStore_internal()
2820 const BYTE* const llCodeTable = seqStorePtr->llCode; in ZSTD_entropyCompressSeqStore_internal()
2821 const BYTE* const mlCodeTable = seqStorePtr->mlCode; in ZSTD_entropyCompressSeqStore_internal()
2829 entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); in ZSTD_entropyCompressSeqStore_internal()
2833 assert(entropyWkspSize >= HUF_WORKSPACE_SIZE); in ZSTD_entropyCompressSeqStore_internal()
2836 { size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); in ZSTD_entropyCompressSeqStore_internal()
2844 &prevEntropy->huf, &nextEntropy->huf, in ZSTD_entropyCompressSeqStore_internal()
2845 cctxParams->cParams.strategy, in ZSTD_entropyCompressSeqStore_internal()
2849 assert(cSize <= dstCapacity); in ZSTD_entropyCompressSeqStore_internal()
2854 RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, in ZSTD_entropyCompressSeqStore_internal()
2864 MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); in ZSTD_entropyCompressSeqStore_internal()
2867 assert(op <= oend); in ZSTD_entropyCompressSeqStore_internal()
2870 ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); in ZSTD_entropyCompressSeqStore_internal()
2871 return (size_t)(op - ostart); in ZSTD_entropyCompressSeqStore_internal()
2877 &prevEntropy->fse, &nextEntropy->fse, in ZSTD_entropyCompressSeqStore_internal()
2889 op, (size_t)(oend - op), in ZSTD_entropyCompressSeqStore_internal()
2897 assert(op <= oend); in ZSTD_entropyCompressSeqStore_internal()
2908 assert(lastCountSize + bitstreamSize == 3); in ZSTD_entropyCompressSeqStore_internal()
2915 DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); in ZSTD_entropyCompressSeqStore_internal()
2916 return (size_t)(op - ostart); in ZSTD_entropyCompressSeqStore_internal()
2947 { size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy); in ZSTD_entropyCompressSeqStore_wExtLitBuffer()
2954 assert(cSize < ZSTD_BLOCKSIZE_MAX); in ZSTD_entropyCompressSeqStore_wExtLitBuffer()
2971 seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart), in ZSTD_entropyCompressSeqStore()
3033 assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); in ZSTD_selectBlockCompressor()
3058 DEBUGLOG(5, "Selecting a row-based matchfinder"); in ZSTD_selectBlockCompressor()
3059 assert(useRowMatchFinder != ZSTD_ps_auto); in ZSTD_selectBlockCompressor()
3060 selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; in ZSTD_selectBlockCompressor()
3064 assert(selectedCompressor != NULL); in ZSTD_selectBlockCompressor()
3071 ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); in ZSTD_storeLastLiterals()
3072 seqStorePtr->lit += lastLLSize; in ZSTD_storeLastLiterals()
3077 ssPtr->lit = ssPtr->litStart; in ZSTD_resetSeqStore()
3078 ssPtr->sequences = ssPtr->sequencesStart; in ZSTD_resetSeqStore()
3079 ssPtr->longLengthType = ZSTD_llt_none; in ZSTD_resetSeqStore()
3083 * Validates and post-processes sequences obtained through the external matchfinder API:
3084 * - Checks whether nbExternalSeqs represents an error condition.
3085 * - Appends a block delimiter to outSeqs if one is not already present.
3087 * Returns the number of sequences after post-processing, or an error code. */
3101 "Got zero sequences from external sequence producer for a non-empty src buffer!" in ZSTD_postProcessSequenceProducerResult()
3110 ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; in ZSTD_postProcessSequenceProducerResult()
3135 * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
3154 const SeqDef* seq = seqStore->sequencesStart; in ZSTD_validateSeqStore()
3155 const SeqDef* const seqEnd = seqStore->sequences; in ZSTD_validateSeqStore()
3156 size_t const matchLenLowerBound = cParams->minMatch == 3 ? 3 : 4; in ZSTD_validateSeqStore()
3159 assert(seqLength.matchLength >= matchLenLowerBound); in ZSTD_validateSeqStore()
3180 ZSTD_MatchState_t* const ms = &zc->blockState.matchState; in ZSTD_buildSeqStore()
3182 assert(srcSize <= ZSTD_BLOCKSIZE_MAX); in ZSTD_buildSeqStore()
3183 /* Assert that we have correctly flushed the ctx params into the ms's copy */ in ZSTD_buildSeqStore()
3184 ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); in ZSTD_buildSeqStore()
3188 if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { in ZSTD_buildSeqStore()
3189 ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); in ZSTD_buildSeqStore()
3191 … ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); in ZSTD_buildSeqStore()
3195 ZSTD_resetSeqStore(&(zc->seqStore)); in ZSTD_buildSeqStore()
3197 ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; in ZSTD_buildSeqStore()
3199 ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; in ZSTD_buildSeqStore()
3203 assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); in ZSTD_buildSeqStore()
3206 { const BYTE* const base = ms->window.base; in ZSTD_buildSeqStore()
3208 const U32 curr = (U32)(istart-base); in ZSTD_buildSeqStore()
3209 … if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ in ZSTD_buildSeqStore()
3210 if (curr > ms->nextToUpdate + 384) in ZSTD_buildSeqStore()
3211 ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); in ZSTD_buildSeqStore()
3219 zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; in ZSTD_buildSeqStore()
3221 if (zc->externSeqStore.pos < zc->externSeqStore.size) { in ZSTD_buildSeqStore()
3222 assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); in ZSTD_buildSeqStore()
3227 ZSTD_hasExtSeqProd(&zc->appliedParams), in ZSTD_buildSeqStore()
3229 … "Long-distance matching with external sequence producer enabled is not currently supported." in ZSTD_buildSeqStore()
3234 ZSTD_ldm_blockCompress(&zc->externSeqStore, in ZSTD_buildSeqStore()
3235 ms, &zc->seqStore, in ZSTD_buildSeqStore()
3236 zc->blockState.nextCBlock->rep, in ZSTD_buildSeqStore()
3237 zc->appliedParams.useRowMatchFinder, in ZSTD_buildSeqStore()
3239 assert(zc->externSeqStore.pos <= zc->externSeqStore.size); in ZSTD_buildSeqStore()
3240 } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { in ZSTD_buildSeqStore()
3246 ZSTD_hasExtSeqProd(&zc->appliedParams), in ZSTD_buildSeqStore()
3248 … "Long-distance matching with external sequence producer enabled is not currently supported." in ZSTD_buildSeqStore()
3251 ldmSeqStore.seq = zc->ldmSequences; in ZSTD_buildSeqStore()
3252 ldmSeqStore.capacity = zc->maxNbLdmSequences; in ZSTD_buildSeqStore()
3254 FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, in ZSTD_buildSeqStore()
3255 &zc->appliedParams.ldmParams, in ZSTD_buildSeqStore()
3260 ms, &zc->seqStore, in ZSTD_buildSeqStore()
3261 zc->blockState.nextCBlock->rep, in ZSTD_buildSeqStore()
3262 zc->appliedParams.useRowMatchFinder, in ZSTD_buildSeqStore()
3264 assert(ldmSeqStore.pos == ldmSeqStore.size); in ZSTD_buildSeqStore()
3265 } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { in ZSTD_buildSeqStore()
3266 assert( in ZSTD_buildSeqStore()
3267 zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize) in ZSTD_buildSeqStore()
3269 assert(zc->appliedParams.extSeqProdFunc != NULL); in ZSTD_buildSeqStore()
3271 { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; in ZSTD_buildSeqStore()
3273 size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( in ZSTD_buildSeqStore()
3274 zc->appliedParams.extSeqProdState, in ZSTD_buildSeqStore()
3275 zc->extSeqBuf, in ZSTD_buildSeqStore()
3276 zc->extSeqBufCapacity, in ZSTD_buildSeqStore()
3279 zc->appliedParams.compressionLevel, in ZSTD_buildSeqStore()
3284 zc->extSeqBuf, in ZSTD_buildSeqStore()
3286 zc->extSeqBufCapacity, in ZSTD_buildSeqStore()
3293 … size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs); in ZSTD_buildSeqStore()
3298 zc->extSeqBuf, nbPostProcessedSeqs, in ZSTD_buildSeqStore()
3300 zc->appliedParams.searchForExternalRepcodes in ZSTD_buildSeqStore()
3304 ms->ldmSeqStore = NULL; in ZSTD_buildSeqStore()
3310 if (!zc->appliedParams.enableMatchFinderFallback) { in ZSTD_buildSeqStore()
3317 zc->appliedParams.cParams.strategy, in ZSTD_buildSeqStore()
3318 zc->appliedParams.useRowMatchFinder, in ZSTD_buildSeqStore()
3320 ms->ldmSeqStore = NULL; in ZSTD_buildSeqStore()
3323 … "External sequence producer returned error code %lu. Falling back to internal parser.", in ZSTD_buildSeqStore()
3326 … lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); in ZSTD_buildSeqStore()
3330 zc->appliedParams.cParams.strategy, in ZSTD_buildSeqStore()
3331 zc->appliedParams.useRowMatchFinder, in ZSTD_buildSeqStore()
3333 ms->ldmSeqStore = NULL; in ZSTD_buildSeqStore()
3334 … lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); in ZSTD_buildSeqStore()
3336 { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; in ZSTD_buildSeqStore()
3337 ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); in ZSTD_buildSeqStore()
3339 ZSTD_validateSeqStore(&zc->seqStore, &zc->appliedParams.cParams); in ZSTD_buildSeqStore()
3345 const SeqDef* inSeqs = seqStore->sequencesStart; in ZSTD_copyBlockSequences()
3346 const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs); in ZSTD_copyBlockSequences()
3347 const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); in ZSTD_copyBlockSequences()
3349 …STD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqSt… in ZSTD_copyBlockSequences()
3358 assert(seqCollector->seqIndex <= seqCollector->maxSequences); in ZSTD_copyBlockSequences()
3360 nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex), in ZSTD_copyBlockSequences()
3375 if (i == seqStore->longLengthPos) { in ZSTD_copyBlockSequences()
3376 if (seqStore->longLengthType == ZSTD_llt_literalLength) { in ZSTD_copyBlockSequences()
3378 } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { in ZSTD_copyBlockSequences()
3386 assert(repcode > 0); in ZSTD_copyBlockSequences()
3389 rawOffset = repcodes.rep[repcode - 1]; in ZSTD_copyBlockSequences()
3392 assert(repcodes.rep[0] > 1); in ZSTD_copyBlockSequences()
3393 rawOffset = repcodes.rep[0] - 1; in ZSTD_copyBlockSequences()
3414 assert(nbInLiterals >= nbOutLiterals); in ZSTD_copyBlockSequences()
3416 const size_t lastLLSize = nbInLiterals - nbOutLiterals; in ZSTD_copyBlockSequences()
3420 assert(nbOutSequences == nbInSequences + 1); in ZSTD_copyBlockSequences()
3422 seqCollector->seqIndex += nbOutSequences; in ZSTD_copyBlockSequences()
3423 assert(seqCollector->seqIndex <= seqCollector->maxSequences); in ZSTD_copyBlockSequences()
3458 zc->seqCollector = seqCollector; in ZSTD_generateSequences()
3465 assert(zc->seqCollector.seqIndex <= ZSTD_sequenceBound(srcSize)); in ZSTD_generateSequences()
3466 return zc->seqCollector.seqIndex; in ZSTD_generateSequences()
3474 if (in != seqsSize - 1) { in ZSTD_mergeBlockDelimiters()
3491 const size_t unrollMask = unrollSize - 1; in ZSTD_isRLE()
3496 if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { in ZSTD_isRLE()
3514 size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); in ZSTD_maybeRLE()
3515 size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); in ZSTD_maybeRLE()
3523 ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; in ZSTD_blockState_confirmRepcodesAndEntropyTables()
3524 bs->prevCBlock = bs->nextCBlock; in ZSTD_blockState_confirmRepcodesAndEntropyTables()
3525 bs->nextCBlock = tmp; in ZSTD_blockState_confirmRepcodesAndEntropyTables()
3561 const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); in ZSTD_buildBlockEntropyStats_literals()
3564 HUF_repeat repeat = prevHuf->repeatMode; in ZSTD_buildBlockEntropyStats_literals()
3571 DEBUGLOG(5, "set_basic - disabled"); in ZSTD_buildBlockEntropyStats_literals()
3572 hufMetadata->hType = set_basic; in ZSTD_buildBlockEntropyStats_literals()
3580 …{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SI… in ZSTD_buildBlockEntropyStats_literals()
3582 DEBUGLOG(5, "set_basic - too small"); in ZSTD_buildBlockEntropyStats_literals()
3583 hufMetadata->hType = set_basic; in ZSTD_buildBlockEntropyStats_literals()
3596 hufMetadata->hType = set_rle; in ZSTD_buildBlockEntropyStats_literals()
3601 DEBUGLOG(5, "set_basic - no gain"); in ZSTD_buildBlockEntropyStats_literals()
3602 hufMetadata->hType = set_basic; in ZSTD_buildBlockEntropyStats_literals()
3608 && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { in ZSTD_buildBlockEntropyStats_literals()
3613 ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); in ZSTD_buildBlockEntropyStats_literals()
3614 …alTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, h… in ZSTD_buildBlockEntropyStats_literals()
3615 assert(huffLog <= LitHufLog); in ZSTD_buildBlockEntropyStats_literals()
3616 { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, in ZSTD_buildBlockEntropyStats_literals()
3624 (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); in ZSTD_buildBlockEntropyStats_literals()
3626 hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), in ZSTD_buildBlockEntropyStats_literals()
3627 (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, in ZSTD_buildBlockEntropyStats_literals()
3632 (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); in ZSTD_buildBlockEntropyStats_literals()
3634 DEBUGLOG(5, "set_repeat - smaller"); in ZSTD_buildBlockEntropyStats_literals()
3636 hufMetadata->hType = set_repeat; in ZSTD_buildBlockEntropyStats_literals()
3640 DEBUGLOG(5, "set_basic - no gains"); in ZSTD_buildBlockEntropyStats_literals()
3642 hufMetadata->hType = set_basic; in ZSTD_buildBlockEntropyStats_literals()
3646 hufMetadata->hType = set_compressed; in ZSTD_buildBlockEntropyStats_literals()
3647 nextHuf->repeatMode = HUF_repeat_check; in ZSTD_buildBlockEntropyStats_literals()
3661 nextEntropy->litlength_repeatMode = FSE_repeat_none; in ZSTD_buildDummySequencesStatistics()
3662 nextEntropy->offcode_repeatMode = FSE_repeat_none; in ZSTD_buildDummySequencesStatistics()
3663 nextEntropy->matchlength_repeatMode = FSE_repeat_none; in ZSTD_buildDummySequencesStatistics()
3681 ZSTD_strategy const strategy = cctxParams->cParams.strategy; in ZSTD_buildBlockEntropyStats_sequences()
3682 size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); in ZSTD_buildBlockEntropyStats_sequences()
3683 BYTE* const ostart = fseMetadata->fseTablesBuffer; in ZSTD_buildBlockEntropyStats_sequences()
3684 BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); in ZSTD_buildBlockEntropyStats_sequences()
3688 size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); in ZSTD_buildBlockEntropyStats_sequences()
3698 fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype; in ZSTD_buildBlockEntropyStats_sequences()
3699 fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype; in ZSTD_buildBlockEntropyStats_sequences()
3700 fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype; in ZSTD_buildBlockEntropyStats_sequences()
3701 fseMetadata->lastCountSize = stats.lastCountSize; in ZSTD_buildBlockEntropyStats_sequences()
3720 size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); in ZSTD_buildBlockEntropyStats()
3721 int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); in ZSTD_buildBlockEntropyStats()
3724 entropyMetadata->hufMetadata.hufDesSize = in ZSTD_buildBlockEntropyStats()
3725 ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, in ZSTD_buildBlockEntropyStats()
3726 &prevEntropy->huf, &nextEntropy->huf, in ZSTD_buildBlockEntropyStats()
3727 &entropyMetadata->hufMetadata, in ZSTD_buildBlockEntropyStats()
3731 …FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals fa… in ZSTD_buildBlockEntropyStats()
3732 entropyMetadata->fseMetadata.fseTablesSize = in ZSTD_buildBlockEntropyStats()
3734 &prevEntropy->fse, &nextEntropy->fse, in ZSTD_buildBlockEntropyStats()
3736 &entropyMetadata->fseMetadata, in ZSTD_buildBlockEntropyStats()
3738 …FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequence… in ZSTD_buildBlockEntropyStats()
3755 if (hufMetadata->hType == set_basic) return litSize; in ZSTD_estimateBlockSize_literal()
3756 else if (hufMetadata->hType == set_rle) return 1; in ZSTD_estimateBlockSize_literal()
3757 else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { in ZSTD_estimateBlockSize_literal()
3760 …{ size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, … in ZSTD_estimateBlockSize_literal()
3761 if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; in ZSTD_estimateBlockSize_literal()
3762 … if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ in ZSTD_estimateBlockSize_literal()
3765 assert(0); /* impossible */ in ZSTD_estimateBlockSize_literal()
3769 /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
3788 assert(max <= defaultMax); in ZSTD_estimateBlockSize_symbolType()
3820 …cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, Max… in ZSTD_estimateBlockSize_sequences()
3821 fseTables->offcodeCTable, NULL, in ZSTD_estimateBlockSize_sequences()
3824 …cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, Max… in ZSTD_estimateBlockSize_sequences()
3825 fseTables->litlengthCTable, LL_bits, in ZSTD_estimateBlockSize_sequences()
3828 …cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, Max… in ZSTD_estimateBlockSize_sequences()
3829 fseTables->matchlengthCTable, ML_bits, in ZSTD_estimateBlockSize_sequences()
3832 if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; in ZSTD_estimateBlockSize_sequences()
3849 &entropy->huf, &entropyMetadata->hufMetadata, in ZSTD_estimateBlockSize()
3852 nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, in ZSTD_estimateBlockSize()
3864 ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3867 &zc->blockState.prevCBlock->entropy, in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3868 &zc->blockState.nextCBlock->entropy, in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3869 &zc->appliedParams, in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3871 zc->tmpWorkspace, zc->tmpWkspSize), ""); in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3873 seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3874 seqStore->ofCode, seqStore->llCode, seqStore->mlCode, in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3875 (size_t)(seqStore->sequences - seqStore->sequencesStart), in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3876 &zc->blockState.nextCBlock->entropy, in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3878 zc->tmpWorkspace, zc->tmpWkspSize, in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3879 (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); in ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()
3886 size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); in ZSTD_countSeqStoreLiteralsBytes()
3889 SeqDef const seq = seqStore->sequencesStart[i]; in ZSTD_countSeqStoreLiteralsBytes()
3891 if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { in ZSTD_countSeqStoreLiteralsBytes()
3901 size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); in ZSTD_countSeqStoreMatchBytes()
3904 SeqDef seq = seqStore->sequencesStart[i]; in ZSTD_countSeqStoreMatchBytes()
3906 if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { in ZSTD_countSeqStoreMatchBytes()
3921 resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; in ZSTD_deriveSeqStoreChunk()
3922 resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); in ZSTD_deriveSeqStoreChunk()
3926 if (originalSeqStore->longLengthType != ZSTD_llt_none) { in ZSTD_deriveSeqStoreChunk()
3927 … if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { in ZSTD_deriveSeqStoreChunk()
3928 resultSeqStore->longLengthType = ZSTD_llt_none; in ZSTD_deriveSeqStoreChunk()
3930 resultSeqStore->longLengthPos -= (U32)startIdx; in ZSTD_deriveSeqStoreChunk()
3933 resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; in ZSTD_deriveSeqStoreChunk()
3934 resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; in ZSTD_deriveSeqStoreChunk()
3935 if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { in ZSTD_deriveSeqStoreChunk()
3937 assert(resultSeqStore->lit == originalSeqStore->lit); in ZSTD_deriveSeqStoreChunk()
3940 resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; in ZSTD_deriveSeqStoreChunk()
3942 resultSeqStore->llCode += startIdx; in ZSTD_deriveSeqStoreChunk()
3943 resultSeqStore->mlCode += startIdx; in ZSTD_deriveSeqStoreChunk()
3944 resultSeqStore->ofCode += startIdx; in ZSTD_deriveSeqStoreChunk()
3954 U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ in ZSTD_resolveRepcodeToRawOffset()
3955 assert(OFFBASE_IS_REPCODE(offBase)); in ZSTD_resolveRepcodeToRawOffset()
3957 assert(ll0); in ZSTD_resolveRepcodeToRawOffset()
3958 /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 in ZSTD_resolveRepcodeToRawOffset()
3965 return rep[0] - 1; in ZSTD_resolveRepcodeToRawOffset()
3978 * Note : this function assumes seq->offBase respects the following numbering scheme :
3980 * 1-3 : repcode 1-3
3988 …U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengt… in ZSTD_seqStore_resolveOffCodes()
3990 SeqDef* const seq = seqStore->sequencesStart + idx; in ZSTD_seqStore_resolveOffCodes()
3991 U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); in ZSTD_seqStore_resolveOffCodes()
3992 U32 const offBase = seq->offBase; in ZSTD_seqStore_resolveOffCodes()
3993 assert(offBase > 0); in ZSTD_seqStore_resolveOffCodes()
3995 U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); in ZSTD_seqStore_resolveOffCodes()
3996 U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); in ZSTD_seqStore_resolveOffCodes()
4002 seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); in ZSTD_seqStore_resolveOffCodes()
4006 …* Decompression repcode history may use modified seq->offset value taken from compression repcode … in ZSTD_seqStore_resolveOffCodes()
4008 ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); in ZSTD_seqStore_resolveOffCodes()
4009 ZSTD_updateRep(cRepcodes->rep, offBase, ll0); in ZSTD_seqStore_resolveOffCodes()
4036 …ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequence… in ZSTD_compressSeqStore_singleBlock()
4040 &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, in ZSTD_compressSeqStore_singleBlock()
4041 &zc->appliedParams, in ZSTD_compressSeqStore_singleBlock()
4042 op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, in ZSTD_compressSeqStore_singleBlock()
4044 zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, in ZSTD_compressSeqStore_singleBlock()
4045 zc->bmi2); in ZSTD_compressSeqStore_singleBlock()
4048 if (!zc->isFirstBlock && in ZSTD_compressSeqStore_singleBlock()
4059 if (zc->seqCollector.collectSequences) { in ZSTD_compressSeqStore_singleBlock()
4060 …FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, seqStore, dRepOriginal.rep), "copyBloc… in ZSTD_compressSeqStore_singleBlock()
4061 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); in ZSTD_compressSeqStore_singleBlock()
4076 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); in ZSTD_compressSeqStore_singleBlock()
4082 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) in ZSTD_compressSeqStore_singleBlock()
4083 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; in ZSTD_compressSeqStore_singleBlock()
4098 * If advantageous to split, then we recurse down the two sub-blocks.
4114 SeqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; in ZSTD_deriveBlockSplitsHelper()
4115 SeqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; in ZSTD_deriveBlockSplitsHelper()
4116 SeqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; in ZSTD_deriveBlockSplitsHelper()
4123 assert(endIdx >= startIdx); in ZSTD_deriveBlockSplitsHelper()
4124 …if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { in ZSTD_deriveBlockSplitsHelper()
4125 DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); in ZSTD_deriveBlockSplitsHelper()
4134 …DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu… in ZSTD_deriveBlockSplitsHelper()
4142 splits->splitLocations[splits->idx] = (U32)midIdx; in ZSTD_deriveBlockSplitsHelper()
4143 splits->idx++; in ZSTD_deriveBlockSplitsHelper()
4149 * Populates a table with intra-block partition indices that can improve compression ratio.
4151 * @return: number of splits made (which equals the size of the partition table - 1).
4163 ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); in ZSTD_deriveBlockSplits()
4185 U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ in ZSTD_compressBlock_splitBlock_internal()
4186 SeqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; in ZSTD_compressBlock_splitBlock_internal()
4187 SeqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; in ZSTD_compressBlock_splitBlock_internal()
4206 ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); in ZSTD_compressBlock_splitBlock_internal()
4207 ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); in ZSTD_compressBlock_splitBlock_internal()
4211 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, in ZSTD_compressBlock_splitBlock_internal()
4212 (unsigned)zc->blockState.matchState.nextToUpdate); in ZSTD_compressBlock_splitBlock_internal()
4216 ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, in ZSTD_compressBlock_splitBlock_internal()
4223 assert(zc->blockSizeMax <= ZSTD_BLOCKSIZE_MAX); in ZSTD_compressBlock_splitBlock_internal()
4224 assert(cSizeSingleBlock <= zc->blockSizeMax + ZSTD_blockHeaderSize); in ZSTD_compressBlock_splitBlock_internal()
4228 ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); in ZSTD_compressBlock_splitBlock_internal()
4238 srcBytes += blockSize - srcBytesTotal; in ZSTD_compressBlock_splitBlock_internal()
4241 ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); in ZSTD_compressBlock_splitBlock_internal()
4255 dstCapacity -= cSizeChunk; in ZSTD_compressBlock_splitBlock_internal()
4258 assert(cSizeChunk <= zc->blockSizeMax + ZSTD_blockHeaderSize); in ZSTD_compressBlock_splitBlock_internal()
4263 ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(Repcodes_t)); in ZSTD_compressBlock_splitBlock_internal()
4275 assert(zc->appliedParams.postBlockSplitter == ZSTD_ps_enable); in ZSTD_compressBlock_splitBlock()
4280 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) in ZSTD_compressBlock_splitBlock()
4281 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; in ZSTD_compressBlock_splitBlock()
4282 …RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"… in ZSTD_compressBlock_splitBlock()
4288 nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); in ZSTD_compressBlock_splitBlock()
4310 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, in ZSTD_compressBlock_internal()
4311 (unsigned)zc->blockState.matchState.nextToUpdate); in ZSTD_compressBlock_internal()
4316 …RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"… in ZSTD_compressBlock_internal()
4322 if (zc->seqCollector.collectSequences) { in ZSTD_compressBlock_internal()
4323 …FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, ZSTD_getSeqStore(zc), zc->blockState.p… in ZSTD_compressBlock_internal()
4324 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); in ZSTD_compressBlock_internal()
4329 cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, in ZSTD_compressBlock_internal()
4330 &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, in ZSTD_compressBlock_internal()
4331 &zc->appliedParams, in ZSTD_compressBlock_internal()
4334 zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, in ZSTD_compressBlock_internal()
4335 zc->bmi2); in ZSTD_compressBlock_internal()
4342 !zc->isFirstBlock && in ZSTD_compressBlock_internal()
4352 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); in ZSTD_compressBlock_internal()
4358 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) in ZSTD_compressBlock_internal()
4359 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; in ZSTD_compressBlock_internal()
4375 !zc->isFirstBlock && in ZSTD_compressBlock_targetCBlockSize_body()
4376 ZSTD_maybeRLE(&zc->seqStore) && in ZSTD_compressBlock_targetCBlockSize_body()
4403 srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); in ZSTD_compressBlock_targetCBlockSize_body()
4406 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); in ZSTD_compressBlock_targetCBlockSize_body()
4428 …(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockSt… in ZSTD_compressBlock_targetCBlockSize()
4434 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) in ZSTD_compressBlock_targetCBlockSize()
4435 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; in ZSTD_compressBlock_targetCBlockSize()
4446 U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); in ZSTD_overflowCorrectIfNeeded()
4447 U32 const maxDist = (U32)1 << params->cParams.windowLog; in ZSTD_overflowCorrectIfNeeded()
4448 …if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)… in ZSTD_overflowCorrectIfNeeded()
4449 U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); in ZSTD_overflowCorrectIfNeeded()
4456 if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; in ZSTD_overflowCorrectIfNeeded()
4457 else ms->nextToUpdate -= correction; in ZSTD_overflowCorrectIfNeeded()
4459 ms->loadedDictEnd = 0; in ZSTD_overflowCorrectIfNeeded()
4460 ms->dictMatchState = NULL; in ZSTD_overflowCorrectIfNeeded()
4477 * require verified savings to allow pre-splitting. in ZSTD_optimalBlockSize()
4489 assert(ZSTD_fast <= strat && strat <= ZSTD_btultra2); in ZSTD_optimalBlockSize()
4492 assert(2 <= splitLevel && splitLevel <= 6); in ZSTD_optimalBlockSize()
4493 splitLevel -= 2; in ZSTD_optimalBlockSize()
4495 return ZSTD_splitBlock(src, blockSizeMax, splitLevel, cctx->tmpWorkspace, cctx->tmpWkspSize); in ZSTD_optimalBlockSize()
4510 size_t blockSizeMax = cctx->blockSizeMax; in ZSTD_compress_frameChunk()
4515 U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; in ZSTD_compress_frameChunk()
4516 S64 savings = (S64)cctx->consumedSrcSize - (S64)cctx->producedCSize; in ZSTD_compress_frameChunk()
4518 assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); in ZSTD_compress_frameChunk()
4521 if (cctx->appliedParams.fParams.checksumFlag && srcSize) in ZSTD_compress_frameChunk()
4522 xxh64_update(&cctx->xxhState, src, srcSize); in ZSTD_compress_frameChunk()
4525 ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; in ZSTD_compress_frameChunk()
4529 cctx->appliedParams.preBlockSplitter_level, in ZSTD_compress_frameChunk()
4530 cctx->appliedParams.cParams.strategy, in ZSTD_compress_frameChunk()
4533 assert(blockSize <= remaining); in ZSTD_compress_frameChunk()
4542 ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); in ZSTD_compress_frameChunk()
4543 …ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchSta… in ZSTD_compress_frameChunk()
4544 … ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); in ZSTD_compress_frameChunk()
4547 if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; in ZSTD_compress_frameChunk()
4550 if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { in ZSTD_compress_frameChunk()
4553 assert(cSize > 0); in ZSTD_compress_frameChunk()
4554 assert(cSize <= blockSize + ZSTD_blockHeaderSize); in ZSTD_compress_frameChunk()
4555 } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { in ZSTD_compress_frameChunk()
4558 assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); in ZSTD_compress_frameChunk()
4561 op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, in ZSTD_compress_frameChunk()
4575 } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ in ZSTD_compress_frameChunk()
4584 * But if the goal is to not expand by more than 3-bytes per 128 KB full block, in ZSTD_compress_frameChunk()
4591 savings += (S64)blockSize - (S64)cSize; in ZSTD_compress_frameChunk()
4594 assert(remaining >= blockSize); in ZSTD_compress_frameChunk()
4595 remaining -= blockSize; in ZSTD_compress_frameChunk()
4597 assert(dstCapacity >= cSize); in ZSTD_compress_frameChunk()
4598 dstCapacity -= cSize; in ZSTD_compress_frameChunk()
4599 cctx->isFirstBlock = 0; in ZSTD_compress_frameChunk()
4604 if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; in ZSTD_compress_frameChunk()
4605 return (size_t)(op-ostart); in ZSTD_compress_frameChunk()
4614 U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ in ZSTD_writeFrameHeader()
4615 … U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ in ZSTD_writeFrameHeader()
4616 U32 const checksumFlag = params->fParams.checksumFlag>0; in ZSTD_writeFrameHeader()
4617 U32 const windowSize = (U32)1 << params->cParams.windowLog; in ZSTD_writeFrameHeader()
4618 U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); in ZSTD_writeFrameHeader()
4619 … BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); in ZSTD_writeFrameHeader()
4620 U32 const fcsCode = params->fParams.contentSizeFlag ? in ZSTD_writeFrameHeader()
4621 …(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ in ZSTD_writeFrameHeader()
4625 assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); in ZSTD_writeFrameHeader()
4627 "dst buf is too small to fit worst-case frame header size."); in ZSTD_writeFrameHeader()
4629 !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); in ZSTD_writeFrameHeader()
4630 if (params->format == ZSTD_f_zstd1) { in ZSTD_writeFrameHeader()
4639 assert(0); /* impossible */ in ZSTD_writeFrameHeader()
4649 assert(0); /* impossible */ in ZSTD_writeFrameHeader()
4652 case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; in ZSTD_writeFrameHeader()
4680 * output an empty Block with end-of-frame mark to complete a frame
4696 assert(cctx->stage == ZSTDcs_init); in ZSTD_referenceExternalSequences()
4697 assert(nbSeq == 0 || cctx->appliedParams.ldmParams.enableLdm != ZSTD_ps_enable); in ZSTD_referenceExternalSequences()
4698 cctx->externSeqStore.seq = seq; in ZSTD_referenceExternalSequences()
4699 cctx->externSeqStore.size = nbSeq; in ZSTD_referenceExternalSequences()
4700 cctx->externSeqStore.capacity = nbSeq; in ZSTD_referenceExternalSequences()
4701 cctx->externSeqStore.pos = 0; in ZSTD_referenceExternalSequences()
4702 cctx->externSeqStore.posInSequence = 0; in ZSTD_referenceExternalSequences()
4711 ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; in ZSTD_compressContinue_internal()
4715 cctx->stage, (unsigned)srcSize); in ZSTD_compressContinue_internal()
4716 RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, in ZSTD_compressContinue_internal()
4719 if (frame && (cctx->stage==ZSTDcs_init)) { in ZSTD_compressContinue_internal()
4720 fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, in ZSTD_compressContinue_internal()
4721 cctx->pledgedSrcSizePlusOne-1, cctx->dictID); in ZSTD_compressContinue_internal()
4723 assert(fhSize <= dstCapacity); in ZSTD_compressContinue_internal()
4724 dstCapacity -= fhSize; in ZSTD_compressContinue_internal()
4726 cctx->stage = ZSTDcs_ongoing; in ZSTD_compressContinue_internal()
4731 if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { in ZSTD_compressContinue_internal()
4732 ms->forceNonContiguous = 0; in ZSTD_compressContinue_internal()
4733 ms->nextToUpdate = ms->window.dictLimit; in ZSTD_compressContinue_internal()
4735 if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { in ZSTD_compressContinue_internal()
4736 ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); in ZSTD_compressContinue_internal()
4742 ms, &cctx->workspace, &cctx->appliedParams, in ZSTD_compressContinue_internal()
4746 DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSizeMax); in ZSTD_compressContinue_internal()
4751 cctx->consumedSrcSize += srcSize; in ZSTD_compressContinue_internal()
4752 cctx->producedCSize += (cSize + fhSize); in ZSTD_compressContinue_internal()
4753 assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); in ZSTD_compressContinue_internal()
4754 if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ in ZSTD_compressContinue_internal()
4755 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); in ZSTD_compressContinue_internal()
4757 cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, in ZSTD_compressContinue_internal()
4760 (unsigned)cctx->pledgedSrcSizePlusOne-1, in ZSTD_compressContinue_internal()
4761 (unsigned)cctx->consumedSrcSize); in ZSTD_compressContinue_internal()
4785 ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; in ZSTD_getBlockSize_deprecated()
4786 assert(!ZSTD_checkCParams(cParams)); in ZSTD_getBlockSize_deprecated()
4787 return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); in ZSTD_getBlockSize_deprecated()
4826 int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; in ZSTD_loadDictionaryContent()
4828 /* Assert that the ms params match the params we're being given */ in ZSTD_loadDictionaryContent()
4829 ZSTD_assertEqualCParams(params->cParams, ms->cParams); in ZSTD_loadDictionaryContent()
4834 * Dictionaries right at the edge will immediately trigger overflow in ZSTD_loadDictionaryContent()
4837 U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; in ZSTD_loadDictionaryContent()
4839 int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); in ZSTD_loadDictionaryContent()
4846 …U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_IND… in ZSTD_loadDictionaryContent()
4848 assert(!loadLdmDict); in ZSTD_loadDictionaryContent()
4853 ip = iend - maxDictSize; in ZSTD_loadDictionaryContent()
4861 assert(ZSTD_window_isEmpty(ms->window)); in ZSTD_loadDictionaryContent()
4862 if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); in ZSTD_loadDictionaryContent()
4864 ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); in ZSTD_loadDictionaryContent()
4866 DEBUGLOG(4, "ZSTD_loadDictionaryContent: useRowMatchFinder=%d", (int)params->useRowMatchFinder); in ZSTD_loadDictionaryContent()
4870 ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); in ZSTD_loadDictionaryContent()
4871 ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); in ZSTD_loadDictionaryContent()
4872 ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); in ZSTD_loadDictionaryContent()
4877 …{ U32 maxDictSize = 1U << MIN(MAX(params->cParams.hashLog + 3, params->cParams.chainLog + 1), 31… in ZSTD_loadDictionaryContent()
4879 ip = iend - maxDictSize; in ZSTD_loadDictionaryContent()
4885 ms->nextToUpdate = (U32)(ip - ms->window.base); in ZSTD_loadDictionaryContent()
4886 ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); in ZSTD_loadDictionaryContent()
4887 ms->forceNonContiguous = params->deterministicRefPrefix; in ZSTD_loadDictionaryContent()
4893 switch(params->cParams.strategy) in ZSTD_loadDictionaryContent()
4902 assert(0); /* shouldn't be called: cparams should've been adjusted. */ in ZSTD_loadDictionaryContent()
4912 assert(srcSize >= HASH_READ_SIZE); in ZSTD_loadDictionaryContent()
4913 if (ms->dedicatedDictSearch) { in ZSTD_loadDictionaryContent()
4914 assert(ms->chainTable != NULL); in ZSTD_loadDictionaryContent()
4915 ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); in ZSTD_loadDictionaryContent()
4917 assert(params->useRowMatchFinder != ZSTD_ps_auto); in ZSTD_loadDictionaryContent()
4918 if (params->useRowMatchFinder == ZSTD_ps_enable) { in ZSTD_loadDictionaryContent()
4919 size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); in ZSTD_loadDictionaryContent()
4920 ZSTD_memset(ms->tagTable, 0, tagTableSize); in ZSTD_loadDictionaryContent()
4921 ZSTD_row_update(ms, iend-HASH_READ_SIZE); in ZSTD_loadDictionaryContent()
4922 DEBUGLOG(4, "Using row-based hash table for lazy dict"); in ZSTD_loadDictionaryContent()
4924 ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); in ZSTD_loadDictionaryContent()
4925 DEBUGLOG(4, "Using chain-based hash table for lazy dict"); in ZSTD_loadDictionaryContent()
4929 assert(0); /* shouldn't be called: cparams should've been adjusted. */ in ZSTD_loadDictionaryContent()
4940 assert(srcSize >= HASH_READ_SIZE); in ZSTD_loadDictionaryContent()
4942 ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); in ZSTD_loadDictionaryContent()
4944 assert(0); /* shouldn't be called: cparams should've been adjusted. */ in ZSTD_loadDictionaryContent()
4949 assert(0); /* not possible : not a valid strategy id */ in ZSTD_loadDictionaryContent()
4952 ms->nextToUpdate = (U32)(iend - ms->window.base); in ZSTD_loadDictionaryContent()
4983 bs->entropy.huf.repeatMode = HUF_repeat_check; in ZSTD_loadCEntropy()
4987 …size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, di… in ZSTD_loadCEntropy()
4988 (size_t)(dictEnd-dictPtr), &hasZeroWeights); in ZSTD_loadCEntropy()
4990 /* We only set the loaded table as valid if it contains all non-zero in ZSTD_loadCEntropy()
4993 bs->entropy.huf.repeatMode = HUF_repeat_valid; in ZSTD_loadCEntropy()
5000 …= FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); in ZSTD_loadCEntropy()
5005 bs->entropy.fse.offcodeCTable, in ZSTD_loadCEntropy()
5015 …ount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); in ZSTD_loadCEntropy()
5019 bs->entropy.fse.matchlengthCTable, in ZSTD_loadCEntropy()
5023 …bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxVa… in ZSTD_loadCEntropy()
5029 …readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); in ZSTD_loadCEntropy()
5033 bs->entropy.fse.litlengthCTable, in ZSTD_loadCEntropy()
5037 …bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, M… in ZSTD_loadCEntropy()
5042 bs->rep[0] = MEM_readLE32(dictPtr+0); in ZSTD_loadCEntropy()
5043 bs->rep[1] = MEM_readLE32(dictPtr+4); in ZSTD_loadCEntropy()
5044 bs->rep[2] = MEM_readLE32(dictPtr+8); in ZSTD_loadCEntropy()
5047 { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); in ZSTD_loadCEntropy()
5049 if (dictContentSize <= ((U32)-1) - 128 KB) { in ZSTD_loadCEntropy()
5054 …bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(off… in ZSTD_loadCEntropy()
5059 RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); in ZSTD_loadCEntropy()
5060 RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); in ZSTD_loadCEntropy()
5063 return (size_t)(dictPtr - (const BYTE*)dict); in ZSTD_loadCEntropy()
5068 * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format
5089 assert(dictSize >= 8); in ZSTD_loadZstdDictionary()
5090 assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); in ZSTD_loadZstdDictionary()
5092 dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); in ZSTD_loadZstdDictionary()
5098 size_t const dictContentSize = (size_t)(dictEnd - dictPtr); in ZSTD_loadZstdDictionary()
5138 assert(0); /* impossible */ in ZSTD_compress_insertDictionary()
5150 * Assumption : either @dict OR @cdict (or none) is non-NULL, never both
5160 size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; in ZSTD_compressBegin_internal()
5161 DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); in ZSTD_compressBegin_internal()
5163 assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); in ZSTD_compressBegin_internal()
5164 assert(!((dict) && (cdict))); /* either dict or cdict, not both */ in ZSTD_compressBegin_internal()
5166 && (cdict->dictContentSize > 0) in ZSTD_compressBegin_internal()
5168 || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER in ZSTD_compressBegin_internal()
5170 || cdict->compressionLevel == 0) in ZSTD_compressBegin_internal()
5171 && (params->attachDictPref != ZSTD_dictForceLoad) ) { in ZSTD_compressBegin_internal()
5180 cctx->blockState.prevCBlock, &cctx->blockState.matchState, in ZSTD_compressBegin_internal()
5181 &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, in ZSTD_compressBegin_internal()
5182 cdict->dictContentSize, cdict->dictContentType, dtlm, in ZSTD_compressBegin_internal()
5183 ZSTD_tfp_forCCtx, cctx->tmpWorkspace) in ZSTD_compressBegin_internal()
5185 cctx->blockState.prevCBlock, &cctx->blockState.matchState, in ZSTD_compressBegin_internal()
5186 &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, in ZSTD_compressBegin_internal()
5187 dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace); in ZSTD_compressBegin_internal()
5189 assert(dictID <= UINT_MAX); in ZSTD_compressBegin_internal()
5190 cctx->dictID = (U32)dictID; in ZSTD_compressBegin_internal()
5191 cctx->dictContentSize = dictContentSize; in ZSTD_compressBegin_internal()
5204 DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); in ZSTD_compressBegin_advanced_internal()
5206 FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); in ZSTD_compressBegin_advanced_internal()
5261 RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); in ZSTD_writeEpilogue()
5264 if (cctx->stage == ZSTDcs_init) { in ZSTD_writeEpilogue()
5265 size_t fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); in ZSTD_writeEpilogue()
5267 dstCapacity -= fhSize; in ZSTD_writeEpilogue()
5269 cctx->stage = ZSTDcs_ongoing; in ZSTD_writeEpilogue()
5272 if (cctx->stage != ZSTDcs_ending) { in ZSTD_writeEpilogue()
5279 dstCapacity -= ZSTD_blockHeaderSize; in ZSTD_writeEpilogue()
5282 if (cctx->appliedParams.fParams.checksumFlag) { in ZSTD_writeEpilogue()
5283 U32 const checksum = (U32) xxh64_digest(&cctx->xxhState); in ZSTD_writeEpilogue()
5290 cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ in ZSTD_writeEpilogue()
5291 return (size_t)(op-ostart); in ZSTD_writeEpilogue()
5309 endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); in ZSTD_compressEnd_public()
5311 assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); in ZSTD_compressEnd_public()
5312 if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ in ZSTD_compressEnd_public()
5313 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); in ZSTD_compressEnd_public()
5316 cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, in ZSTD_compressEnd_public()
5319 (unsigned)cctx->pledgedSrcSizePlusOne-1, in ZSTD_compressEnd_public()
5320 (unsigned)cctx->consumedSrcSize); in ZSTD_compressEnd_public()
5342 ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); in ZSTD_compress_advanced()
5347 &cctx->simpleApiParams); in ZSTD_compress_advanced()
5373 assert(params.fParams.contentSizeFlag == 1); in ZSTD_compress_usingDict()
5374 …ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEV… in ZSTD_compress_usingDict()
5377 …ss_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); in ZSTD_compress_usingDict()
5386 assert(cctx != NULL); in ZSTD_compressCCtx()
5415 * in case we are using DDS with row-hash. */ in ZSTD_estimateCDictSize_advanced()
5433 return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) in ZSTD_sizeof_CDict()
5434 + ZSTD_cwksp_sizeof(&cdict->workspace); in ZSTD_sizeof_CDict()
5445 assert(!ZSTD_checkCParams(params.cParams)); in ZSTD_initCDict_internal()
5446 cdict->matchState.cParams = params.cParams; in ZSTD_initCDict_internal()
5447 cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; in ZSTD_initCDict_internal()
5449 cdict->dictContent = dictBuffer; in ZSTD_initCDict_internal()
5451 …void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, siz… in ZSTD_initCDict_internal()
5453 cdict->dictContent = internalBuffer; in ZSTD_initCDict_internal()
5456 cdict->dictContentSize = dictSize; in ZSTD_initCDict_internal()
5457 cdict->dictContentType = dictContentType; in ZSTD_initCDict_internal()
5459 … cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); in ZSTD_initCDict_internal()
5463 ZSTD_reset_compressedBlockState(&cdict->cBlockState); in ZSTD_initCDict_internal()
5465 &cdict->matchState, in ZSTD_initCDict_internal()
5466 &cdict->workspace, in ZSTD_initCDict_internal()
5478 &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, in ZSTD_initCDict_internal()
5479 ¶ms, cdict->dictContent, cdict->dictContentSize, in ZSTD_initCDict_internal()
5480 dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); in ZSTD_initCDict_internal()
5482 assert(dictID <= (size_t)(U32)-1); in ZSTD_initCDict_internal()
5483 cdict->dictID = (U32)dictID; in ZSTD_initCDict_internal()
5519 assert(cdict != NULL); in ZSTD_createCDict_advanced_internal()
5520 ZSTD_cwksp_move(&cdict->workspace, &ws); in ZSTD_createCDict_advanced_internal()
5521 cdict->customMem = customMem; in ZSTD_createCDict_advanced_internal()
5522 cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ in ZSTD_createCDict_advanced_internal()
5523 cdict->useRowMatchFinder = useRowMatchFinder; in ZSTD_createCDict_advanced_internal()
5570 /* Fall back to non-DDSS params */ in ZSTD_createCDict_advanced2()
5603 cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; in ZSTD_createCDict()
5614 cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; in ZSTD_createCDict_byReference()
5621 { ZSTD_customMem const cMem = cdict->customMem; in ZSTD_freeCDict()
5622 int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); in ZSTD_freeCDict()
5623 ZSTD_cwksp_free(&cdict->workspace, cMem); in ZSTD_freeCDict()
5634 * Provided pointer must 8-bytes aligned.
5663 if ((size_t)workspace & 7) return NULL; /* 8-aligned */ in ZSTD_initStaticCDict()
5670 ZSTD_cwksp_move(&cdict->workspace, &ws); in ZSTD_initStaticCDict()
5678 cdict->useRowMatchFinder = useRowMatchFinder; in ZSTD_initStaticCDict()
5679 cdict->compressionLevel = ZSTD_NO_CLEVEL; in ZSTD_initStaticCDict()
5692 assert(cdict != NULL); in ZSTD_getCParamsFromCDict()
5693 return cdict->matchState.cParams; in ZSTD_getCParamsFromCDict()
5699 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
5703 return cdict->dictID; in ZSTD_getDictID_fromCDict()
5721 … || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER in ZSTD_compressBegin_usingCDict_internal()
5723 || cdict->compressionLevel == 0 ) ? in ZSTD_compressBegin_usingCDict_internal()
5725 : ZSTD_getCParams(cdict->compressionLevel, in ZSTD_compressBegin_usingCDict_internal()
5727 cdict->dictContentSize); in ZSTD_compressBegin_usingCDict_internal()
5728 ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); in ZSTD_compressBegin_usingCDict_internal()
5736 U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; in ZSTD_compressBegin_usingCDict_internal()
5842 return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; in ZSTD_CStreamOutSize()
5880 assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); in ZSTD_initCStream_internal()
5881 zcs->requestedParams = *params; in ZSTD_initCStream_internal()
5882 assert(!((dict) && (cdict))); /* either dict or cdict, not both */ in ZSTD_initCStream_internal()
5902 zcs->requestedParams.fParams = fParams; in ZSTD_initCStream_usingCDict_advanced()
5934 ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); in ZSTD_initCStream_advanced()
5976 if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { in ZSTD_nextInputSizeHint()
5977 return cctx->blockSizeMax - cctx->stableIn_notConsumed; in ZSTD_nextInputSizeHint()
5979 assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); in ZSTD_nextInputSizeHint()
5980 { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; in ZSTD_nextInputSizeHint()
5981 if (hintInSize==0) hintInSize = cctx->blockSizeMax; in ZSTD_nextInputSizeHint()
5994 const char* const istart = (assert(input != NULL), (const char*)input->src); in ZSTD_compressStream_generic()
5995 const char* const iend = (istart != NULL) ? istart + input->size : istart; in ZSTD_compressStream_generic()
5996 const char* ip = (istart != NULL) ? istart + input->pos : istart; in ZSTD_compressStream_generic()
5997 char* const ostart = (assert(output != NULL), (char*)output->dst); in ZSTD_compressStream_generic()
5998 char* const oend = (ostart != NULL) ? ostart + output->size : ostart; in ZSTD_compressStream_generic()
5999 char* op = (ostart != NULL) ? ostart + output->pos : ostart; in ZSTD_compressStream_generic()
6003 … "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); in ZSTD_compressStream_generic()
6004 assert(zcs != NULL); in ZSTD_compressStream_generic()
6005 if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { in ZSTD_compressStream_generic()
6006 assert(input->pos >= zcs->stableIn_notConsumed); in ZSTD_compressStream_generic()
6007 input->pos -= zcs->stableIn_notConsumed; in ZSTD_compressStream_generic()
6008 if (ip) ip -= zcs->stableIn_notConsumed; in ZSTD_compressStream_generic()
6009 zcs->stableIn_notConsumed = 0; in ZSTD_compressStream_generic()
6011 if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { in ZSTD_compressStream_generic()
6012 assert(zcs->inBuff != NULL); in ZSTD_compressStream_generic()
6013 assert(zcs->inBuffSize > 0); in ZSTD_compressStream_generic()
6015 if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { in ZSTD_compressStream_generic()
6016 assert(zcs->outBuff != NULL); in ZSTD_compressStream_generic()
6017 assert(zcs->outBuffSize > 0); in ZSTD_compressStream_generic()
6019 if (input->src == NULL) assert(input->size == 0); in ZSTD_compressStream_generic()
6020 assert(input->pos <= input->size); in ZSTD_compressStream_generic()
6021 if (output->dst == NULL) assert(output->size == 0); in ZSTD_compressStream_generic()
6022 assert(output->pos <= output->size); in ZSTD_compressStream_generic()
6023 assert((U32)flushMode <= (U32)ZSTD_e_end); in ZSTD_compressStream_generic()
6026 switch(zcs->streamStage) in ZSTD_compressStream_generic()
6033 … && ( (size_t)(oend-op) >= ZSTD_compressBound((size_t)(iend-ip)) /* Enough output space */ in ZSTD_compressStream_generic()
6034 …|| zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooS… in ZSTD_compressStream_generic()
6035 && (zcs->inBuffPos == 0) ) { in ZSTD_compressStream_generic()
6038 op, (size_t)(oend-op), in ZSTD_compressStream_generic()
6039 ip, (size_t)(iend-ip)); in ZSTD_compressStream_generic()
6044 zcs->frameEnded = 1; in ZSTD_compressStream_generic()
6049 if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { in ZSTD_compressStream_generic()
6050 size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; in ZSTD_compressStream_generic()
6052 zcs->inBuff + zcs->inBuffPos, toLoad, in ZSTD_compressStream_generic()
6053 ip, (size_t)(iend-ip)); in ZSTD_compressStream_generic()
6054 zcs->inBuffPos += loaded; in ZSTD_compressStream_generic()
6057 && (zcs->inBuffPos < zcs->inBuffTarget) ) { in ZSTD_compressStream_generic()
6062 && (zcs->inBuffPos == zcs->inToCompress) ) { in ZSTD_compressStream_generic()
6067 assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); in ZSTD_compressStream_generic()
6069 && ( (size_t)(iend - ip) < zcs->blockSizeMax) ) { in ZSTD_compressStream_generic()
6071 zcs->stableIn_notConsumed = (size_t)(iend - ip); in ZSTD_compressStream_generic()
6083 { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); in ZSTD_compressStream_generic()
6086 size_t oSize = (size_t)(oend-op); in ZSTD_compressStream_generic()
6087 size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress in ZSTD_compressStream_generic()
6088 : MIN((size_t)(iend - ip), zcs->blockSizeMax); in ZSTD_compressStream_generic()
6089 … if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) in ZSTD_compressStream_generic()
6092 cDst = zcs->outBuff, oSize = zcs->outBuffSize; in ZSTD_compressStream_generic()
6097 zcs->inBuff + zcs->inToCompress, iSize) : in ZSTD_compressStream_generic()
6099 zcs->inBuff + zcs->inToCompress, iSize); in ZSTD_compressStream_generic()
6101 zcs->frameEnded = lastBlock; in ZSTD_compressStream_generic()
6103 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSizeMax; in ZSTD_compressStream_generic()
6104 if (zcs->inBuffTarget > zcs->inBuffSize) in ZSTD_compressStream_generic()
6105 zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSizeMax; in ZSTD_compressStream_generic()
6107 (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); in ZSTD_compressStream_generic()
6109 assert(zcs->inBuffTarget <= zcs->inBuffSize); in ZSTD_compressStream_generic()
6110 zcs->inToCompress = zcs->inBuffPos; in ZSTD_compressStream_generic()
6119 zcs->frameEnded = lastBlock; in ZSTD_compressStream_generic()
6120 if (lastBlock) assert(ip == iend); in ZSTD_compressStream_generic()
6124 if (zcs->frameEnded) { in ZSTD_compressStream_generic()
6131 zcs->outBuffContentSize = cSize; in ZSTD_compressStream_generic()
6132 zcs->outBuffFlushedSize = 0; in ZSTD_compressStream_generic()
6133 zcs->streamStage = zcss_flush; /* pass-through to flush stage */ in ZSTD_compressStream_generic()
6138 assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); in ZSTD_compressStream_generic()
6139 { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; in ZSTD_compressStream_generic()
6140 size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), in ZSTD_compressStream_generic()
6141 zcs->outBuff + zcs->outBuffFlushedSize, toFlush); in ZSTD_compressStream_generic()
6143 (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); in ZSTD_compressStream_generic()
6146 zcs->outBuffFlushedSize += flushed; in ZSTD_compressStream_generic()
6149 assert(op==oend); in ZSTD_compressStream_generic()
6153 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; in ZSTD_compressStream_generic()
6154 if (zcs->frameEnded) { in ZSTD_compressStream_generic()
6160 zcs->streamStage = zcss_load; in ZSTD_compressStream_generic()
6165 assert(0); in ZSTD_compressStream_generic()
6169 input->pos = (size_t)(ip - istart); in ZSTD_compressStream_generic()
6170 output->pos = (size_t)(op - ostart); in ZSTD_compressStream_generic()
6171 if (zcs->frameEnded) return 0; in ZSTD_compressStream_generic()
6194 if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { in ZSTD_setBufferExpectations()
6195 cctx->expectedInBuffer = *input; in ZSTD_setBufferExpectations()
6197 if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { in ZSTD_setBufferExpectations()
6198 cctx->expectedOutBufferSize = output->size - output->pos; in ZSTD_setBufferExpectations()
6210 if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { in ZSTD_checkBufferStability()
6211 ZSTD_inBuffer const expect = cctx->expectedInBuffer; in ZSTD_checkBufferStability()
6212 if (expect.src != input->src || expect.pos != input->pos) in ZSTD_checkBufferStability()
6216 if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { in ZSTD_checkBufferStability()
6217 size_t const outBufferSize = output->size - output->pos; in ZSTD_checkBufferStability()
6218 if (cctx->expectedOutBufferSize != outBufferSize) in ZSTD_checkBufferStability()
6233 ZSTD_CCtx_params params = cctx->requestedParams; in ZSTD_CCtx_init_compressStream2()
6234 ZSTD_prefixDict const prefixDict = cctx->prefixDict; in ZSTD_CCtx_init_compressStream2()
6236 ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ in ZSTD_CCtx_init_compressStream2()
6237 assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ in ZSTD_CCtx_init_compressStream2()
6238 if (cctx->cdict && !cctx->localDict.cdict) { in ZSTD_CCtx_init_compressStream2()
6243 params.compressionLevel = cctx->cdict->compressionLevel; in ZSTD_CCtx_init_compressStream2()
6246 …if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSi… in ZSTD_CCtx_init_compressStream2()
6250 : (cctx->cdict ? cctx->cdict->dictContentSize : 0); in ZSTD_CCtx_init_compressStream2()
6251 …STD_CParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne… in ZSTD_CCtx_init_compressStream2()
6253 ¶ms, cctx->pledgedSrcSizePlusOne-1, in ZSTD_CCtx_init_compressStream2()
6264 { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; in ZSTD_CCtx_init_compressStream2()
6265 assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); in ZSTD_CCtx_init_compressStream2()
6268 cctx->cdict, in ZSTD_CCtx_init_compressStream2()
6271 assert(cctx->appliedParams.nbWorkers == 0); in ZSTD_CCtx_init_compressStream2()
6272 cctx->inToCompress = 0; in ZSTD_CCtx_init_compressStream2()
6273 cctx->inBuffPos = 0; in ZSTD_CCtx_init_compressStream2()
6274 if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { in ZSTD_CCtx_init_compressStream2()
6276 * it would require to add a 3-bytes null block to end frame in ZSTD_CCtx_init_compressStream2()
6278 cctx->inBuffTarget = cctx->blockSizeMax + (cctx->blockSizeMax == pledgedSrcSize); in ZSTD_CCtx_init_compressStream2()
6280 cctx->inBuffTarget = 0; in ZSTD_CCtx_init_compressStream2()
6282 cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; in ZSTD_CCtx_init_compressStream2()
6283 cctx->streamStage = zcss_load; in ZSTD_CCtx_init_compressStream2()
6284 cctx->frameEnded = 0; in ZSTD_CCtx_init_compressStream2()
6298 RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); in ZSTD_compressStream2()
6299 RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); in ZSTD_compressStream2()
6301 assert(cctx != NULL); in ZSTD_compressStream2()
6304 if (cctx->streamStage == zcss_init) { in ZSTD_compressStream2()
6305 size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ in ZSTD_compressStream2()
6306 size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; in ZSTD_compressStream2()
6307 …if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across in… in ZSTD_compressStream2()
6310 if (cctx->stableIn_notConsumed) { /* not the first time */ in ZSTD_compressStream2()
6312 …RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stable… in ZSTD_compressStream2()
6313 …RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stabl… in ZSTD_compressStream2()
6316 input->pos = input->size; in ZSTD_compressStream2()
6318 cctx->expectedInBuffer = *input; in ZSTD_compressStream2()
6320 cctx->stableIn_notConsumed += inputSize; in ZSTD_compressStream2()
6322 …return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce… in ZSTD_compressStream2()
6334 return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ in ZSTD_compressStream2()
6363 ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; in ZSTD_compress2()
6364 ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; in ZSTD_compress2()
6368 cctx->requestedParams.inBufferMode = ZSTD_bm_stable; in ZSTD_compress2()
6369 cctx->requestedParams.outBufferMode = ZSTD_bm_stable; in ZSTD_compress2()
6377 cctx->requestedParams.inBufferMode = originalInBufferMode; in ZSTD_compress2()
6378 cctx->requestedParams.outBufferMode = originalOutBufferMode; in ZSTD_compress2()
6382 assert(oPos == dstCapacity); in ZSTD_compress2()
6385 assert(iPos == srcSize); /* all input is expected consumed */ in ZSTD_compress2()
6420 offBase = REPCODE_TO_OFFBASE(2 - ll0); in ZSTD_finalizeOffBase()
6422 offBase = REPCODE_TO_OFFBASE(3 - ll0); in ZSTD_finalizeOffBase()
6423 } else if (ll0 && rawOffset == rep[0] - 1) { in ZSTD_finalizeOffBase()
6442 U32 idx = seqPos->idx; in ZSTD_transferSequences_wBlockDelim()
6451 if (cctx->cdict) { in ZSTD_transferSequences_wBlockDelim()
6452 dictSize = (U32)cctx->cdict->dictContentSize; in ZSTD_transferSequences_wBlockDelim()
6453 } else if (cctx->prefixDict.dict) { in ZSTD_transferSequences_wBlockDelim()
6454 dictSize = (U32)cctx->prefixDict.dictSize; in ZSTD_transferSequences_wBlockDelim()
6458 ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); in ZSTD_transferSequences_wBlockDelim()
6473 if (cctx->appliedParams.validateSequences) { in ZSTD_transferSequences_wBlockDelim()
6474 seqPos->posInSrc += litLength + matchLength; in ZSTD_transferSequences_wBlockDelim()
6475 … FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, in ZSTD_transferSequences_wBlockDelim()
6476 seqPos->posInSrc, in ZSTD_transferSequences_wBlockDelim()
6477 cctx->appliedParams.cParams.windowLog, dictSize, in ZSTD_transferSequences_wBlockDelim()
6478 ZSTD_hasExtSeqProd(&cctx->appliedParams)), in ZSTD_transferSequences_wBlockDelim()
6481 RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, in ZSTD_transferSequences_wBlockDelim()
6483 ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); in ZSTD_transferSequences_wBlockDelim()
6489 assert(externalRepSearch != ZSTD_ps_auto); in ZSTD_transferSequences_wBlockDelim()
6490 assert(idx >= startIdx); in ZSTD_transferSequences_wBlockDelim()
6493 U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ in ZSTD_transferSequences_wBlockDelim()
6496 rep[2] = inSeqs[lastSeqIdx - 2].offset; in ZSTD_transferSequences_wBlockDelim()
6497 rep[1] = inSeqs[lastSeqIdx - 1].offset; in ZSTD_transferSequences_wBlockDelim()
6501 rep[1] = inSeqs[lastSeqIdx - 1].offset; in ZSTD_transferSequences_wBlockDelim()
6504 assert(lastSeqIdx == startIdx); in ZSTD_transferSequences_wBlockDelim()
6511 ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); in ZSTD_transferSequences_wBlockDelim()
6515 ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); in ZSTD_transferSequences_wBlockDelim()
6517 seqPos->posInSrc += inSeqs[idx].litLength; in ZSTD_transferSequences_wBlockDelim()
6520 seqPos->idx = idx+1; in ZSTD_transferSequences_wBlockDelim()
6542 U32 idx = seqPos->idx; in ZSTD_transferSequences_noDelim()
6543 U32 startPosInSequence = seqPos->posInSequence; in ZSTD_transferSequences_noDelim()
6544 U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; in ZSTD_transferSequences_noDelim()
6556 if (cctx->cdict) { in ZSTD_transferSequences_noDelim()
6557 dictSize = cctx->cdict->dictContentSize; in ZSTD_transferSequences_noDelim()
6558 } else if (cctx->prefixDict.dict) { in ZSTD_transferSequences_noDelim()
6559 dictSize = cctx->prefixDict.dictSize; in ZSTD_transferSequences_noDelim()
6565 ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); in ZSTD_transferSequences_noDelim()
6576 startPosInSequence -= litLength; in ZSTD_transferSequences_noDelim()
6578 matchLength -= startPosInSequence; in ZSTD_transferSequences_noDelim()
6580 litLength -= startPosInSequence; in ZSTD_transferSequences_noDelim()
6583 endPosInSequence -= currSeq.litLength + currSeq.matchLength; in ZSTD_transferSequences_noDelim()
6589 … currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); in ZSTD_transferSequences_noDelim()
6592 litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; in ZSTD_transferSequences_noDelim()
6593 firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; in ZSTD_transferSequences_noDelim()
6594 … if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { in ZSTD_transferSequences_noDelim()
6596 … U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; in ZSTD_transferSequences_noDelim()
6597 if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { in ZSTD_transferSequences_noDelim()
6599 … endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; in ZSTD_transferSequences_noDelim()
6600 … bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; in ZSTD_transferSequences_noDelim()
6601 firstHalfMatchLength -= bytesAdjustment; in ZSTD_transferSequences_noDelim()
6604 … /* Flag that we split the last match - after storing the sequence, exit the loop, in ZSTD_transferSequences_noDelim()
6613 bytesAdjustment = endPosInSequence - currSeq.litLength; in ZSTD_transferSequences_noDelim()
6628 if (cctx->appliedParams.validateSequences) { in ZSTD_transferSequences_noDelim()
6629 seqPos->posInSrc += litLength + matchLength; in ZSTD_transferSequences_noDelim()
6630 …F_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->… in ZSTD_transferSequences_noDelim()
6631 … cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), in ZSTD_transferSequences_noDelim()
6635 RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, in ZSTD_transferSequences_noDelim()
6637 ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); in ZSTD_transferSequences_noDelim()
6643 … assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); in ZSTD_transferSequences_noDelim()
6644 seqPos->idx = idx; in ZSTD_transferSequences_noDelim()
6645 seqPos->posInSequence = endPosInSequence; in ZSTD_transferSequences_noDelim()
6646 ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); in ZSTD_transferSequences_noDelim()
6648 iend -= bytesAdjustment; in ZSTD_transferSequences_noDelim()
6651 U32 const lastLLSize = (U32)(iend - ip); in ZSTD_transferSequences_noDelim()
6652 assert(ip <= iend); in ZSTD_transferSequences_noDelim()
6654 ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); in ZSTD_transferSequences_noDelim()
6655 seqPos->posInSrc += lastLLSize; in ZSTD_transferSequences_noDelim()
6658 return (size_t)(iend-istart); in ZSTD_transferSequences_noDelim()
6674 assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, (int)mode)); in ZSTD_selectSequenceCopier()
6678 assert(mode == ZSTD_sf_noBlockDelimiters); in ZSTD_selectSequenceCopier()
6693 assert(spos <= inSeqsSize); in blockSize_explicitDelimiter()
6719 assert(mode == ZSTD_sf_explicitBlockDelimiters); in determine_blockSize()
6730 /* Compress all provided sequences, block-by-block.
6747 …ZSTD_SequenceCopier_f const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDe… in ZSTD_compressSequences_internal()
6756 dstCapacity -= ZSTD_blockHeaderSize; in ZSTD_compressSequences_internal()
6763 size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, in ZSTD_compressSequences_internal()
6764 cctx->blockSizeMax, remaining, in ZSTD_compressSequences_internal()
6768 assert(blockSize <= remaining); in ZSTD_compressSequences_internal()
6769 ZSTD_resetSeqStore(&cctx->seqStore); in ZSTD_compressSequences_internal()
6774 cctx->appliedParams.searchForExternalRepcodes); in ZSTD_compressSequences_internal()
6787 remaining -= blockSize; in ZSTD_compressSequences_internal()
6788 dstCapacity -= cBlockSize; in ZSTD_compressSequences_internal()
6793 compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, in ZSTD_compressSequences_internal()
6794 … &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, in ZSTD_compressSequences_internal()
6795 &cctx->appliedParams, in ZSTD_compressSequences_internal()
6796 … op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, in ZSTD_compressSequences_internal()
6798 … cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, in ZSTD_compressSequences_internal()
6799 cctx->bmi2); in ZSTD_compressSequences_internal()
6803 if (!cctx->isFirstBlock && in ZSTD_compressSequences_internal()
6804 ZSTD_maybeRLE(&cctx->seqStore) && in ZSTD_compressSequences_internal()
6825 ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); in ZSTD_compressSequences_internal()
6826 if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) in ZSTD_compressSequences_internal()
6827 cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; in ZSTD_compressSequences_internal()
6843 remaining -= blockSize; in ZSTD_compressSequences_internal()
6844 dstCapacity -= cBlockSize; in ZSTD_compressSequences_internal()
6845 cctx->isFirstBlock = 0; in ZSTD_compressSequences_internal()
6864 assert(cctx != NULL); in ZSTD_compressSequences()
6869 &cctx->appliedParams, srcSize, cctx->dictID); in ZSTD_compressSequences()
6871 assert(frameHeaderSize <= dstCapacity); in ZSTD_compressSequences()
6872 dstCapacity -= frameHeaderSize; in ZSTD_compressSequences()
6875 if (cctx->appliedParams.fParams.checksumFlag && srcSize) { in ZSTD_compressSequences()
6876 xxh64_update(&cctx->xxhState, src, srcSize); in ZSTD_compressSequences()
6886 assert(cBlocksSize <= dstCapacity); in ZSTD_compressSequences()
6887 dstCapacity -= cBlocksSize; in ZSTD_compressSequences()
6891 if (cctx->appliedParams.fParams.checksumFlag) { in ZSTD_compressSequences()
6892 U32 const checksum = (U32) xxh64_digest(&cctx->xxhState); in ZSTD_compressSequences()
6910 * - offset -> offBase = offset + 2
6911 * - litLength -> (U16) litLength
6912 * - matchLength -> (U16)(matchLength - 3)
6913 * - rep is ignored
6931 * For each 128-bit half: (offset+2, litLength+0, matchLength-3, rep+0) in convertSequences_noRepcodes()
6934 ZSTD_REP_NUM, 0, -MINMATCH, 0, /* for sequence i */ in convertSequences_noRepcodes()
6935 ZSTD_REP_NUM, 0, -MINMATCH, 0 /* for sequence i+1 */ in convertSequences_noRepcodes()
6942 * shuffle mask for byte-level rearrangement in each 128-bit half: in convertSequences_noRepcodes()
6944 * Input layout (after addition) per 128-bit half: in convertSequences_noRepcodes()
6992 /* Add {2, 0, -3, 0} in each 128-bit half */ in convertSequences_noRepcodes()
7009 /* Permute 64-bit lanes => move Lane2 down into Lane1. */ in convertSequences_noRepcodes()
7020 * - offset 0..7 => seq0 (offBase, litLength, mlBase) in convertSequences_noRepcodes()
7021 * - offset 8..15 => seq1 (offBase, litLength, mlBase) in convertSequences_noRepcodes()
7031 assert(longLen == 0); in convertSequences_noRepcodes()
7035 assert(longLen == 0); in convertSequences_noRepcodes()
7039 assert(longLen == 0); in convertSequences_noRepcodes()
7043 assert(longLen == 0); in convertSequences_noRepcodes()
7052 assert(i == nbSequences - 1); in convertSequences_noRepcodes()
7055 dstSeqs[i].mlBase = (U16)(inSeqs[i].matchLength - MINMATCH); in convertSequences_noRepcodes()
7058 assert(longLen == 0); in convertSequences_noRepcodes()
7062 assert(longLen == 0); in convertSequences_noRepcodes()
7072 * it's not useful to develop and maintain code for older pre-AVX2 platforms */
7086 dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH); in convertSequences_noRepcodes()
7089 assert(longLen == 0); in convertSequences_noRepcodes()
7093 assert(longLen == 0); in convertSequences_noRepcodes()
7107 * It may be re-inserted later.
7118 RETURN_ERROR_IF(nbSequences >= cctx->seqStore.maxNbSeq, externalSequences_invalid, in ZSTD_convertBlockSequences()
7121 ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); in ZSTD_convertBlockSequences()
7124 assert(nbSequences >= 1); in ZSTD_convertBlockSequences()
7125 assert(inSeqs[nbSequences-1].matchLength == 0); in ZSTD_convertBlockSequences()
7126 assert(inSeqs[nbSequences-1].offset == 0); in ZSTD_convertBlockSequences()
7130 …size_t const longl = convertSequences_noRepcodes(cctx->seqStore.sequencesStart, inSeqs, nbSequence… in ZSTD_convertBlockSequences()
7131 cctx->seqStore.sequences = cctx->seqStore.sequencesStart + nbSequences-1; in ZSTD_convertBlockSequences()
7134 assert(cctx->seqStore.longLengthType == ZSTD_llt_none); in ZSTD_convertBlockSequences()
7135 if (longl <= nbSequences-1) { in ZSTD_convertBlockSequences()
7136 DEBUGLOG(5, "long match length detected at pos %zu", longl-1); in ZSTD_convertBlockSequences()
7137 cctx->seqStore.longLengthType = ZSTD_llt_matchLength; in ZSTD_convertBlockSequences()
7138 cctx->seqStore.longLengthPos = (U32)(longl-1); in ZSTD_convertBlockSequences()
7140 DEBUGLOG(5, "long literals length detected at pos %zu", longl-nbSequences); in ZSTD_convertBlockSequences()
7141 assert(longl <= 2* (nbSequences-1)); in ZSTD_convertBlockSequences()
7142 cctx->seqStore.longLengthType = ZSTD_llt_literalLength; in ZSTD_convertBlockSequences()
7143 cctx->seqStore.longLengthPos = (U32)(longl-(nbSequences-1)-1); in ZSTD_convertBlockSequences()
7147 for (seqNb = 0; seqNb < nbSequences - 1 ; seqNb++) { in ZSTD_convertBlockSequences()
7154 ZSTD_storeSeqOnly(&cctx->seqStore, litLength, offBase, matchLength); in ZSTD_convertBlockSequences()
7164 U32 lastSeqIdx = (U32)nbSequences - 2; /* index of last full sequence */ in ZSTD_convertBlockSequences()
7165 rep[2] = inSeqs[lastSeqIdx - 2].offset; in ZSTD_convertBlockSequences()
7166 rep[1] = inSeqs[lastSeqIdx - 1].offset; in ZSTD_convertBlockSequences()
7173 assert(nbSequences == 2); in ZSTD_convertBlockSequences()
7180 ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); in ZSTD_convertBlockSequences()
7191 __m256i sumVec = zeroVec; /* accumulates match+lit in 32-bit lanes */ in ZSTD_get1BlockSummary()
7244 assert(seqs); in ZSTD_get1BlockSummary()
7249 assert(seqs[n].offset == 0); in ZSTD_get1BlockSummary()
7277 int const repcodeResolution = (cctx->appliedParams.searchForExternalRepcodes == ZSTD_ps_enable); in ZSTD_compressSequencesAndLiterals_internal()
7278 assert(cctx->appliedParams.searchForExternalRepcodes != ZSTD_ps_auto); in ZSTD_compressSequencesAndLiterals_internal()
7281 … RETURN_ERROR_IF(nbSequences == 0, externalSequences_invalid, "Requires at least 1 end-of-block"); in ZSTD_compressSequencesAndLiterals_internal()
7289 dstCapacity -= ZSTD_blockHeaderSize; in ZSTD_compressSequencesAndLiterals_internal()
7298 assert(block.nbSequences <= nbSequences); in ZSTD_compressSequencesAndLiterals_internal()
7300 ZSTD_resetSeqStore(&cctx->seqStore); in ZSTD_compressSequencesAndLiterals_internal()
7307 nbSequences -= block.nbSequences; in ZSTD_compressSequencesAndLiterals_internal()
7308 remaining -= block.blockSize; in ZSTD_compressSequencesAndLiterals_internal()
7318 … op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, in ZSTD_compressSequencesAndLiterals_internal()
7320 &cctx->seqStore, in ZSTD_compressSequencesAndLiterals_internal()
7321 … &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, in ZSTD_compressSequencesAndLiterals_internal()
7322 &cctx->appliedParams, in ZSTD_compressSequencesAndLiterals_internal()
7323 … cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, in ZSTD_compressSequencesAndLiterals_internal()
7324 cctx->bmi2); in ZSTD_compressSequencesAndLiterals_internal()
7327 if (compressedSeqsSize > cctx->blockSizeMax) compressedSeqsSize = 0; in ZSTD_compressSequencesAndLiterals_internal()
7329 litSize -= block.litSize; in ZSTD_compressSequencesAndLiterals_internal()
7344 assert(compressedSeqsSize > 1); /* no RLE */ in ZSTD_compressSequencesAndLiterals_internal()
7346 ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); in ZSTD_compressSequencesAndLiterals_internal()
7347 if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) in ZSTD_compressSequencesAndLiterals_internal()
7348 cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; in ZSTD_compressSequencesAndLiterals_internal()
7359 dstCapacity -= cBlockSize; in ZSTD_compressSequencesAndLiterals_internal()
7360 cctx->isFirstBlock = 0; in ZSTD_compressSequencesAndLiterals_internal()
7364 assert(nbSequences == 0); in ZSTD_compressSequencesAndLiterals_internal()
7387 assert(cctx != NULL); in ZSTD_compressSequencesAndLiterals()
7389 …er is not large enough: must be at least 8 bytes larger than litSize (risk of read out-of-bound)"); in ZSTD_compressSequencesAndLiterals()
7393 if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) { in ZSTD_compressSequencesAndLiterals()
7396 if (cctx->appliedParams.validateSequences) { in ZSTD_compressSequencesAndLiterals()
7399 if (cctx->appliedParams.fParams.checksumFlag) { in ZSTD_compressSequencesAndLiterals()
7405 &cctx->appliedParams, decompressedSize, cctx->dictID); in ZSTD_compressSequencesAndLiterals()
7407 assert(frameHeaderSize <= dstCapacity); in ZSTD_compressSequencesAndLiterals()
7408 dstCapacity -= frameHeaderSize; in ZSTD_compressSequencesAndLiterals()
7419 assert(cBlocksSize <= dstCapacity); in ZSTD_compressSequencesAndLiterals()
7420 dstCapacity -= cBlocksSize; in ZSTD_compressSequencesAndLiterals()
7432 const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); in inBuffer_forEndFlush()
7433 return stableInput ? zcs->expectedInBuffer : nullInput; in inBuffer_forEndFlush()
7450 if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ in ZSTD_endStream()
7452 { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; in ZSTD_endStream()
7453 …size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag… in ZSTD_endStream()
7461 /*-===== Pre-defined compression levels =====-*/
7465 int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } in ZSTD_minCLevel()
7492 return (cParams->strategy >= ZSTD_greedy) in ZSTD_dedicatedDictSearch_isSupported()
7493 && (cParams->strategy <= ZSTD_lazy2) in ZSTD_dedicatedDictSearch_isSupported()
7494 && (cParams->hashLog > cParams->chainLog) in ZSTD_dedicatedDictSearch_isSupported()
7495 && (cParams->chainLog <= 24); in ZSTD_dedicatedDictSearch_isSupported()
7505 switch (cParams->strategy) { in ZSTD_dedicatedDictSearch_revertCParams()
7512 cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; in ZSTD_dedicatedDictSearch_revertCParams()
7513 if (cParams->hashLog < ZSTD_HASHLOG_MIN) { in ZSTD_dedicatedDictSearch_revertCParams()
7514 cParams->hashLog = ZSTD_HASHLOG_MIN; in ZSTD_dedicatedDictSearch_revertCParams()
7536 assert(0); in ZSTD_getCParamRowSize()
7568 cp.targetLength = (unsigned)(-clampedCompressionLevel); in ZSTD_getCParams_internal()
7615 assert(zc != NULL); in ZSTD_registerSequenceProducer()
7617 &zc->requestedParams, extSeqProdState, extSeqProdFunc in ZSTD_registerSequenceProducer()
7626 assert(params != NULL); in ZSTD_CCtxParams_registerSequenceProducer()
7628 params->extSeqProdFunc = extSeqProdFunc; in ZSTD_CCtxParams_registerSequenceProducer()
7629 params->extSeqProdState = extSeqProdState; in ZSTD_CCtxParams_registerSequenceProducer()
7631 params->extSeqProdFunc = NULL; in ZSTD_CCtxParams_registerSequenceProducer()
7632 params->extSeqProdState = NULL; in ZSTD_CCtxParams_registerSequenceProducer()