AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize()

Browse Source 
it complements ZSTD_estimateCCtxSize()
for the special case of ZSTD_initCStream_usingDict()
This commit is contained in:
Yann Collet 2017-05-08 17:51:49 -07:00
parent 7855366598
commit a1d6704d7f
5 changed files with 57 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
doc/zstd_manual.html
View File

@ -372,15 +372,17 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
</b><p> Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
`frameContentSize` is an optional parameter, provide `0` if unknown
</p></pre><BR>
<pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
</b><p> Create a ZSTD compression context using external alloc and free functions
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
</b><p> Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
Set streaming to 1 if the CCtx will be used for streaming (CStream).
Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size
</p></pre><BR>
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
</b><p> amount of used memory is variable, depending primarily on compression level
</p></pre><BR>
@ -406,6 +408,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
</b><p> Estimate amount of memory that will be needed to create a dictionary with following arguments
Note : if dictionary is created "byReference", reduce this amount by dictSize
</p></pre><BR>
<pre><b>size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
</b><p> Gives the amount of memory used by a given ZSTD_sizeof_CDict
</p></pre><BR>
@ -476,6 +483,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
</b><p> Create a ZSTD_DDict using external alloc and free, optionally by reference
</p></pre><BR>
<pre><b>size_t ZSTD_estimateDDictSize(size_t dictSize);
</b><p> Estimate amount of memory that will be needed to create a dictionary for decompression.
Note : if dictionary is created "byReference", reduce this amount by dictSize
</p></pre><BR>
<pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
</b><p> Gives the amount of memory used by a given ZSTD_DDict
</p></pre><BR>
@ -509,7 +521,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); </b>/**< same as ZSTD_sizeof_CCtx */<b>
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */<b>
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>

8
lib/compress/zstd_compress.c
View File

@ -2984,6 +2984,14 @@ struct ZSTD_CDict_s {
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
/*! ZSTD_estimateCDictSize() :
* Estimate amount of memory that will be needed to create a dictionary with following arguments */
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
{
cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0);
}
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support sizeof on NULL */

8
lib/decompress/zstd_decompress.c
View File

@ -1969,6 +1969,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
}
}
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
size_t ZSTD_estimateDDictSize(size_t dictSize)
{
return dictSize + sizeof(ZSTD_DDict);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */

24
lib/zstd.h
View File

@ -458,17 +458,17 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
/***************************************
* Advanced compression functions
***************************************/
/*! ZSTD_estimateCCtxSize() :
* Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
* Set streaming to 1 if the CCtx will be used for streaming (CStream)
* Note : this function is currently unable to estimate additional memory allocation needed to create an internal CDict
* which can only happen when starting with ZSTD_initCStream_usingDict() */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
/*! ZSTD_createCCtx_advanced() :
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_estimateCCtxSize() :
* Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
* Set streaming to 1 if the CCtx will be used for streaming (CStream).
* Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
* Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
/*! ZSTD_sizeofCCtx() :
* amount of used memory is variable, depending primarily on compression level */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@ -493,6 +493,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
/*! ZSTD_estimateCDictSize() :
* Estimate amount of memory that will be needed to create a dictionary with following arguments
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
/*! ZSTD_sizeof_CDict() :
* Gives the amount of memory used by a given ZSTD_sizeof_CDict */
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
@ -564,6 +569,11 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
/*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);

6
tests/fuzzer.c
View File

@ -404,6 +404,12 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
}
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem customMem = { NULL, NULL, NULL };