5  * This source code is licensed under both the BSD-style license (found in the
8  * You may select, at your option, one of the above-listed licenses.
14 /*-*************************************
20 /*-*************************************
39 /*-*************************************
51  * expect a well-formed caller to free this.
67  * - These different internal datastructures have different setup requirements:
69  *   - The static objects need to be cleared once and can then be trivially
72  *   - Various buffers don't need to be initialized at all--they are always
73  *     written into before they're read.
75  *   - The matchstate tables have a unique requirement that they don't need
83  * - These buffers also have different alignment requirements.
85  * - We would like to reuse the objects in the workspace for multiple
89  * - We would like to be able to efficiently reuse the workspace across
99  * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
104  * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict,
110  * - Fixed size objects: these are fixed-size, fixed-count objects that are
112  *   control how they're initialized separately from the broader ZSTD_CCtx.
114  *   - Entropy Workspace
115  *   - 2 x ZSTD_compressedBlockState_t
116  *   - CDict dictionary contents
118  * - Tables: these are any of several different datastructures (hash tables,
120  *   uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
121  *   Their sizes depend on the cparams. These tables are 64-byte aligned.
123  * - Aligned: these buffers are used for various purposes that require 4 byte
124  *   alignment, but don't require any initialization before they're used. These
127  * - Buffers: these buffers are used for various purposes that don't require
128  *   any alignment or initialization before they're used. This means they can
157 /*-*************************************
165     assert(ws->workspace <= ws->objectEnd);  in ZSTD_cwksp_assert_internal_consistency()
166     assert(ws->objectEnd <= ws->tableEnd);  in ZSTD_cwksp_assert_internal_consistency()
167     assert(ws->objectEnd <= ws->tableValidEnd);  in ZSTD_cwksp_assert_internal_consistency()
168     assert(ws->tableEnd <= ws->allocStart);  in ZSTD_cwksp_assert_internal_consistency()
169     assert(ws->tableValidEnd <= ws->allocStart);  in ZSTD_cwksp_assert_internal_consistency()
170     assert(ws->allocStart <= ws->workspaceEnd);  in ZSTD_cwksp_assert_internal_consistency()
177     size_t const mask = align - 1;  in ZSTD_cwksp_align()
230     size_t const alignBytesMask = alignBytes - 1;  in ZSTD_cwksp_bytes_to_align_ptr()
231     size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;  in ZSTD_cwksp_bytes_to_align_ptr()
247     void* const alloc = (BYTE*)ws->allocStart - bytes;  in ZSTD_cwksp_reserve_internal_buffer_space()
248     void* const bottom = ws->tableEnd;  in ZSTD_cwksp_reserve_internal_buffer_space()
250         alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);  in ZSTD_cwksp_reserve_internal_buffer_space()
255         ws->allocFailed = 1;  in ZSTD_cwksp_reserve_internal_buffer_space()
260     if (alloc < ws->tableValidEnd) {  in ZSTD_cwksp_reserve_internal_buffer_space()
261         ws->tableValidEnd = alloc;  in ZSTD_cwksp_reserve_internal_buffer_space()
263     ws->allocStart = alloc;  in ZSTD_cwksp_reserve_internal_buffer_space()
269  * cwksp initialization must necessarily go through each phase in order.
275     assert(phase >= ws->phase);  in ZSTD_cwksp_internal_advance_phase()
276     if (phase > ws->phase) {  in ZSTD_cwksp_internal_advance_phase()
278         if (ws->phase < ZSTD_cwksp_alloc_buffers &&  in ZSTD_cwksp_internal_advance_phase()
280             ws->tableValidEnd = ws->objectEnd;  in ZSTD_cwksp_internal_advance_phase()
284         if (ws->phase < ZSTD_cwksp_alloc_aligned &&  in ZSTD_cwksp_internal_advance_phase()
288 …ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BY…  in ZSTD_cwksp_internal_advance_phase()
290 …ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power…  in ZSTD_cwksp_internal_advance_phase()
292 …                        memory_allocation, "aligned phase - alignment initial allocation failed!");  in ZSTD_cwksp_internal_advance_phase()
295                 void* const alloc = ws->objectEnd;  in ZSTD_cwksp_internal_advance_phase()
299                 RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,  in ZSTD_cwksp_internal_advance_phase()
300                                 "table phase - alignment initial allocation failed!");  in ZSTD_cwksp_internal_advance_phase()
301                 ws->objectEnd = objectEnd;  in ZSTD_cwksp_internal_advance_phase()
302                 ws->tableEnd = objectEnd;  /* table area starts being empty */  in ZSTD_cwksp_internal_advance_phase()
303                 if (ws->tableValidEnd < ws->tableEnd) {  in ZSTD_cwksp_internal_advance_phase()
304                     ws->tableValidEnd = ws->tableEnd;  in ZSTD_cwksp_internal_advance_phase()
306         ws->phase = phase;  in ZSTD_cwksp_internal_advance_phase()
317     return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);  in ZSTD_cwksp_owns_buffer()
353     assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);  in ZSTD_cwksp_reserve_aligned()
359  * their values remain constrained, allowing us to re-use them without
360  * memset()-ing them.
372     alloc = ws->tableEnd;  in ZSTD_cwksp_reserve_table()
374     top = ws->allocStart;  in ZSTD_cwksp_reserve_table()
377         alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);  in ZSTD_cwksp_reserve_table()
378     assert((bytes & (sizeof(U32)-1)) == 0);  in ZSTD_cwksp_reserve_table()
383         ws->allocFailed = 1;  in ZSTD_cwksp_reserve_table()
386     ws->tableEnd = end;  in ZSTD_cwksp_reserve_table()
389     assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);  in ZSTD_cwksp_reserve_table()
390     assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);  in ZSTD_cwksp_reserve_table()
396  * Note : should happen only once, at workspace first initialization
401     void* alloc = ws->objectEnd;  in ZSTD_cwksp_reserve_object()
407         alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);  in ZSTD_cwksp_reserve_object()
412     if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) {  in ZSTD_cwksp_reserve_object()
414         ws->allocFailed = 1;  in ZSTD_cwksp_reserve_object()
417     ws->objectEnd = end;  in ZSTD_cwksp_reserve_object()
418     ws->tableEnd = end;  in ZSTD_cwksp_reserve_object()
419     ws->tableValidEnd = end;  in ZSTD_cwksp_reserve_object()
430     assert(ws->tableValidEnd >= ws->objectEnd);  in ZSTD_cwksp_mark_tables_dirty()
431     assert(ws->tableValidEnd <= ws->allocStart);  in ZSTD_cwksp_mark_tables_dirty()
432     ws->tableValidEnd = ws->objectEnd;  in ZSTD_cwksp_mark_tables_dirty()
438     assert(ws->tableValidEnd >= ws->objectEnd);  in ZSTD_cwksp_mark_tables_clean()
439     assert(ws->tableValidEnd <= ws->allocStart);  in ZSTD_cwksp_mark_tables_clean()
440     if (ws->tableValidEnd < ws->tableEnd) {  in ZSTD_cwksp_mark_tables_clean()
441         ws->tableValidEnd = ws->tableEnd;  in ZSTD_cwksp_mark_tables_clean()
451     assert(ws->tableValidEnd >= ws->objectEnd);  in ZSTD_cwksp_clean_tables()
452     assert(ws->tableValidEnd <= ws->allocStart);  in ZSTD_cwksp_clean_tables()
453     if (ws->tableValidEnd < ws->tableEnd) {  in ZSTD_cwksp_clean_tables()
454         ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);  in ZSTD_cwksp_clean_tables()
467     ws->tableEnd = ws->objectEnd;  in ZSTD_cwksp_clear_tables()
480     ws->tableEnd = ws->objectEnd;  in ZSTD_cwksp_clear()
481     ws->allocStart = ws->workspaceEnd;  in ZSTD_cwksp_clear()
482     ws->allocFailed = 0;  in ZSTD_cwksp_clear()
483     if (ws->phase > ZSTD_cwksp_alloc_buffers) {  in ZSTD_cwksp_clear()
484         ws->phase = ZSTD_cwksp_alloc_buffers;  in ZSTD_cwksp_clear()
496     assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */  in ZSTD_cwksp_init()
497     ws->workspace = start;  in ZSTD_cwksp_init()
498     ws->workspaceEnd = (BYTE*)start + size;  in ZSTD_cwksp_init()
499     ws->objectEnd = ws->workspace;  in ZSTD_cwksp_init()
500     ws->tableValidEnd = ws->objectEnd;  in ZSTD_cwksp_init()
501     ws->phase = ZSTD_cwksp_alloc_objects;  in ZSTD_cwksp_init()
502     ws->isStatic = isStatic;  in ZSTD_cwksp_init()
504     ws->workspaceOversizedDuration = 0;  in ZSTD_cwksp_init()
517     void *ptr = ws->workspace;  in ZSTD_cwksp_free()
525  * is left in an invalid state (src must be re-init()'ed before it's used again).
533     return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);  in ZSTD_cwksp_sizeof()
537     return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace)  in ZSTD_cwksp_used()
538          + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart);  in ZSTD_cwksp_used()
542     return ws->allocFailed;  in ZSTD_cwksp_reserve_failed()
545 /*-*************************************
562 …return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63…  in ZSTD_cwksp_estimated_space_within_bounds()
568     return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);  in ZSTD_cwksp_available_space()
582         && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION;  in ZSTD_cwksp_check_wasteful()
588         ws->workspaceOversizedDuration++;  in ZSTD_cwksp_bump_oversized_duration()
590         ws->workspaceOversizedDuration = 0;  in ZSTD_cwksp_bump_oversized_duration()