Merge pull request #737 from facebook/advancedAPI2
simplified API for *initStatic*()
This commit is contained in:
Yann Collet
GitHub
commit
ad5f29918e
@ -332,7 +332,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
</b></pre><BR>
|
||||
<pre><b>typedef struct {
|
||||
unsigned long long frameContentSize;
|
||||
unsigned windowSize;
|
||||
size_t windowSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
} ZSTD_frameHeader;
|
||||
@ -389,6 +389,12 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||
however it does mean that all frame data must be present and valid.
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
||||
</b><p> `src` should point to the start of a ZSTD frame
|
||||
`srcSize` must be >= ZSTD_frameHeaderSize_prefix.
|
||||
@return : size of the Frame Header
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter13"></a><h2>Context memory usage</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
||||
@ -401,25 +407,41 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
Object memory usage can evolve if it's re-used multiple times.
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
|
||||
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
||||
size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
|
||||
size_t ZSTD_estimateDCtxSize(void);
|
||||
</b><p> These functions make it possible to estimate memory usage
|
||||
of a future target object, before its allocation,
|
||||
given a set of parameters, which vary depending on target object.
|
||||
The objective is to guide decision before allocation.
|
||||
of a future {D,C}Ctx, before its creation.
|
||||
ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one.
|
||||
It will also consider src size to be arbitrarily "large", which is worst case.
|
||||
If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
|
||||
ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
||||
Note : CCtx estimation is only correct for single-threaded compression
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
|
||||
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
|
||||
</b><p> Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
|
||||
an internal ?Dict will be created, which size is not estimated here.
|
||||
<pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel);
|
||||
size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
|
||||
size_t ZSTD_estimateDStreamSize(size_t windowSize);
|
||||
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
|
||||
</b><p> ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one.
|
||||
It will also consider src size to be arbitrarily "large", which is worst case.
|
||||
If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
|
||||
ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
||||
Note : CStream estimation is only correct for single-threaded compression.
|
||||
ZSTD_DStream memory budget depends on window Size.
|
||||
This information can be passed manually, using ZSTD_estimateDStreamSize,
|
||||
or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
|
||||
Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
|
||||
an internal ?Dict will be created, which additional size is not estimated here.
|
||||
In this case, get total size by adding ZSTD_estimate?DictSize
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
|
||||
<pre><b>size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
|
||||
size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
|
||||
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
|
||||
</b><p> Note : dictionary created "byReference" are smaller
|
||||
</b><p> ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
|
||||
ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
|
||||
Note : dictionary created "byReference" are smaller
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||
@ -606,6 +628,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
|
||||
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
|
||||
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b>
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */<b>
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
@ -624,6 +647,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
|
||||
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticDCtx() */<b>
|
||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
||||
@ -678,16 +702,18 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
||||
It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
|
||||
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID used.
|
||||
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
||||
such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID in use.
|
||||
(Note : content size is optional, it may not be present. 0 means : content size unknown).
|
||||
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
|
||||
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
|
||||
Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
|
||||
Frame parameters are extracted from the beginning of the compressed frame.
|
||||
Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
|
||||
Each application can set its own limit, depending on local restrictions.
|
||||
For extended interoperability, it is recommended to support windowSize of at least 8 MB.
|
||||
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
||||
Data fragment must be large enough to ensure successful decoding.
|
||||
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTD_isError().
|
||||
|
||||
@ -731,7 +757,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
|
||||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
|
||||
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
@ -532,7 +532,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
|
||||
size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams)
|
||||
{
|
||||
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
|
||||
U32 const divider = (cParams.searchLength==3) ? 3 : 4;
|
||||
@ -558,9 +558,15 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
|
||||
return sizeof(ZSTD_CCtx) + neededSpace;
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
|
||||
size_t ZSTD_estimateCCtxSize(int compressionLevel)
|
||||
{
|
||||
size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
|
||||
return ZSTD_estimateCCtxSize_advanced(cParams);
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams)
|
||||
{
|
||||
size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced(cParams);
|
||||
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
|
||||
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
|
||||
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
|
||||
@ -569,6 +575,11 @@ size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
|
||||
return CCtxSize + streamingSize;
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateCStreamSize(int compressionLevel) {
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
|
||||
return ZSTD_estimateCStreamSize_advanced(cParams);
|
||||
}
|
||||
|
||||
|
||||
static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
|
||||
ZSTD_compressionParameters cParams2)
|
||||
@ -3350,16 +3361,22 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
|
||||
|
||||
/* ===== Dictionary API ===== */
|
||||
|
||||
/*! ZSTD_estimateCDictSize() :
|
||||
/*! ZSTD_estimateCDictSize_advanced() :
|
||||
* Estimate amount of memory that will be needed to create a dictionary with following arguments */
|
||||
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
|
||||
size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference)
|
||||
{
|
||||
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
|
||||
DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize(cParams));
|
||||
return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
|
||||
DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams));
|
||||
return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams)
|
||||
+ (byReference ? 0 : dictSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
return ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
|
||||
}
|
||||
|
||||
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
|
||||
{
|
||||
if (cdict==NULL) return 0; /* support sizeof on NULL */
|
||||
@ -3482,7 +3499,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
|
||||
unsigned byReference, ZSTD_dictMode_e dictMode,
|
||||
ZSTD_compressionParameters cParams)
|
||||
{
|
||||
size_t const cctxSize = ZSTD_estimateCCtxSize(cParams);
|
||||
size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams);
|
||||
size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize)
|
||||
+ cctxSize;
|
||||
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
|
||||
@ -3577,6 +3594,11 @@ ZSTD_CStream* ZSTD_createCStream(void)
|
||||
return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize)
|
||||
{
|
||||
return ZSTD_initStaticCCtx(workspace, workspaceSize);
|
||||
}
|
||||
|
||||
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
|
||||
{ /* CStream and CCtx are now same object */
|
||||
return ZSTD_createCCtx_advanced(customMem);
|
||||
@ -3970,30 +3992,30 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
|
||||
int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
|
||||
|
||||
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
|
||||
{ /* "default" */
|
||||
{ /* "default" - guarantees a monotonically increasing memory budget */
|
||||
/* W, C, H, S, L, TL, strat */
|
||||
{ 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */
|
||||
{ 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */
|
||||
{ 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */
|
||||
{ 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/
|
||||
{ 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/
|
||||
{ 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */
|
||||
{ 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
|
||||
{ 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */
|
||||
{ 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */
|
||||
{ 20, 17, 18, 1, 5, 16, ZSTD_dfast }, /* level 4 */
|
||||
{ 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */
|
||||
{ 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
|
||||
{ 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */
|
||||
{ 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */
|
||||
{ 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */
|
||||
{ 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */
|
||||
{ 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */
|
||||
{ 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */
|
||||
{ 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */
|
||||
{ 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */
|
||||
{ 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */
|
||||
{ 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
|
||||
{ 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
|
||||
{ 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */
|
||||
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */
|
||||
{ 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */
|
||||
{ 26, 26, 23, 7, 3,256, ZSTD_btultra }, /* level 21 */
|
||||
{ 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */
|
||||
{ 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */
|
||||
},
|
||||
{ /* for srcSize <= 256 KB */
|
||||
@ -4076,6 +4098,43 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
|
||||
},
|
||||
};
|
||||
|
||||
/* This function just controls
|
||||
* the monotonic memory budget increase of ZSTD_defaultCParameters[0].
|
||||
* Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
|
||||
*/
|
||||
MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
|
||||
{
|
||||
# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
|
||||
# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
|
||||
# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
|
||||
assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \
|
||||
assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
|
||||
}
|
||||
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
|
||||
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
|
||||
assert(ZSTD_maxCLevel()==22);
|
||||
}
|
||||
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
|
||||
* Size values are optional, provide 0 if not known or unused */
|
||||
@ -4084,6 +4143,15 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
|
||||
size_t const addedSize = srcSizeHint ? 0 : 500;
|
||||
U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
|
||||
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
|
||||
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
|
||||
static int g_monotonicTest = 1;
|
||||
if (g_monotonicTest) {
|
||||
ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget();
|
||||
g_monotonicTest=0;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */
|
||||
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
|
||||
{ ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
|
||||
|
||||
@ -1282,7 +1282,7 @@ static size_t ZSTD_decompressSequencesLong(
|
||||
const BYTE* const base = (const BYTE*) (dctx->base);
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
unsigned const windowSize = dctx->fParams.windowSize;
|
||||
unsigned const windowSize32 = (unsigned)dctx->fParams.windowSize;
|
||||
int nbSeq;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
@ -1312,13 +1312,13 @@ static size_t ZSTD_decompressSequencesLong(
|
||||
|
||||
/* prepare in advance */
|
||||
for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb<seqAdvance; seqNb++) {
|
||||
sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
|
||||
sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize32);
|
||||
}
|
||||
if (seqNb<seqAdvance) return ERROR(corruption_detected);
|
||||
|
||||
/* decode and decompress */
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
|
||||
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
|
||||
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize32);
|
||||
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
ZSTD_PREFETCH(sequence.match);
|
||||
@ -2143,6 +2143,11 @@ ZSTD_DStream* ZSTD_createDStream(void)
|
||||
return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
|
||||
{
|
||||
return ZSTD_initStaticDCtx(workspace, workspaceSize);
|
||||
}
|
||||
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
return ZSTD_createDCtx_advanced(customMem);
|
||||
@ -2214,15 +2219,23 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
|
||||
return ZSTD_sizeof_DCtx(zds);
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
|
||||
size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
||||
{
|
||||
size_t const windowSize = fHeader.windowSize;
|
||||
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||
size_t const inBuffSize = blockSize; /* no block can be larger */
|
||||
size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
|
||||
return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
|
||||
}
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_frameHeader fh;
|
||||
size_t const err = ZSTD_getFrameHeader(&fh, src, srcSize);
|
||||
if (ZSTD_isError(err)) return err;
|
||||
if (err>0) return ERROR(srcSize_wrong);
|
||||
return ZSTD_estimateDStreamSize(fh.windowSize);
|
||||
}
|
||||
|
||||
|
||||
/* ***** Decompression ***** */
|
||||
|
||||
|
||||
54
lib/zstd.h
54
lib/zstd.h
@ -412,7 +412,7 @@ typedef struct {
|
||||
|
||||
typedef struct {
|
||||
unsigned long long frameContentSize;
|
||||
unsigned windowSize;
|
||||
size_t windowSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
} ZSTD_frameHeader;
|
||||
@ -495,23 +495,39 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
|
||||
/*! ZSTD_estimate*() :
|
||||
* These functions make it possible to estimate memory usage
|
||||
* of a future target object, before its allocation,
|
||||
* given a set of parameters, which vary depending on target object.
|
||||
* The objective is to guide decision before allocation.
|
||||
* of a future {D,C}Ctx, before its creation.
|
||||
* ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
|
||||
* It will also consider src size to be arbitrarily "large", which is worst case.
|
||||
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
|
||||
* ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
||||
* Note : CCtx estimation is only correct for single-threaded compression */
|
||||
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
|
||||
|
||||
/*! ZSTD_estimate?StreamSize() :
|
||||
* ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
|
||||
* It will also consider src size to be arbitrarily "large", which is worst case.
|
||||
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
|
||||
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
||||
* Note : CStream estimation is only correct for single-threaded compression.
|
||||
* ZSTD_DStream memory budget depends on window Size.
|
||||
* This information can be passed manually, using ZSTD_estimateDStreamSize,
|
||||
* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
|
||||
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
|
||||
* an internal ?Dict will be created, which size is not estimated here.
|
||||
* an internal ?Dict will be created, which additional size is not estimated here.
|
||||
* In this case, get total size by adding ZSTD_estimate?DictSize */
|
||||
ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_estimate?DictSize() :
|
||||
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
|
||||
* ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
|
||||
* Note : dictionary created "byReference" are smaller */
|
||||
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
|
||||
|
||||
|
||||
@ -707,6 +723,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
@ -727,6 +744,7 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
|
||||
/*===== Advanced Streaming decompression functions =====*/
|
||||
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
|
||||
@ -793,16 +811,18 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
||||
It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
|
||||
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID used.
|
||||
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
||||
such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID in use.
|
||||
(Note : content size is optional, it may not be present. 0 means : content size unknown).
|
||||
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
|
||||
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
|
||||
Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
|
||||
Frame parameters are extracted from the beginning of the compressed frame.
|
||||
Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
|
||||
Each application can set its own limit, depending on local restrictions.
|
||||
For extended interoperability, it is recommended to support windowSize of at least 8 MB.
|
||||
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
||||
Data fragment must be large enough to ensure successful decoding.
|
||||
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTD_isError().
|
||||
|
||||
@ -847,7 +867,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||
*/
|
||||
|
||||
/*===== Buffer-less streaming decompression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
|
||||
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
@ -193,8 +193,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
/* Static CCtx tests */
|
||||
#define STATIC_CCTX_LEVEL 3
|
||||
DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
|
||||
size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
|
||||
{ size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
|
||||
void* const staticCCtxBuffer = malloc(staticCCtxSize);
|
||||
size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
|
||||
void* const staticDCtxBuffer = malloc(staticDCtxSize);
|
||||
@ -502,7 +501,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1 /*byReference*/);
|
||||
size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 1 /*byReference*/);
|
||||
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
|
||||
}
|
||||
|
||||
@ -535,7 +534,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
size_t const cdictSize = ZSTD_estimateCDictSize(cParams, dictSize, 0);
|
||||
size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
|
||||
void* const cdictBuffer = malloc(cdictSize);
|
||||
if (cdictBuffer==NULL) goto _output_error;
|
||||
{ ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
|
||||
|
||||
@ -390,8 +390,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
||||
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
|
||||
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
||||
|
||||
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params);
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params);
|
||||
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_advanced(params);
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_advanced(winners[cLevel].params);
|
||||
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
||||
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
||||
|
||||
|
||||
@ -210,9 +210,9 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
/* context size functions */
|
||||
DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
|
||||
size_t const s = ZSTD_estimateCStreamSize(cParams)
|
||||
size_t const s = ZSTD_estimateCStreamSize_advanced(cParams)
|
||||
/* uses ZSTD_initCStream_usingDict() */
|
||||
+ ZSTD_estimateCDictSize(cParams, dictSize, 0);
|
||||
+ ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
|
||||
if (ZSTD_isError(s)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
|
||||
}
|
||||
@ -285,8 +285,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
|
||||
size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
|
||||
if (gfhError!=0) goto _output_error;
|
||||
DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
|
||||
{ size_t const s = ZSTD_estimateDStreamSize(fhi)
|
||||
DISPLAYLEVEL(5, " (windowSize : %u) ", (U32)fhi.windowSize);
|
||||
{ size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
|
||||
/* uses ZSTD_initDStream_usingDict() */
|
||||
+ ZSTD_estimateDDictSize(dictSize, 0);
|
||||
if (ZSTD_isError(s)) goto _output_error;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user