AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updated ZSTD_estimate?DictSize() to pass parameter byReference

Browse Source 
resulting ?Dict object is smaller when created byReference.
Seems better than a documentation note.
This commit is contained in:
Yann Collet 2017-05-25 15:07:37 -07:00
parent 0fdc71c3dc
commit 25989e361c
6 changed files with 53 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
doc/zstd_manual.html
View File

@ -412,9 +412,9 @@ size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
In this case, get additional size by using ZSTD_estimate?DictSize
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
size_t ZSTD_estimateDDictSize(size_t dictSize);
</b><p> Note : if dictionary is created "byReference", reduce estimation by dictSize
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
</b><p> Note : dictionary created "byReference" are smaller
</p></pre><BR>
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
@ -718,6 +718,21 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
</b><p> Create a ZSTD_DDict using external alloc and free, optionally by reference
</p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference);
</b><p> Generate a digested dictionary in provided memory area.
workspace: The memory area to emplace the dictionary into.
Provided pointer must 8-bytes aligned.
It must outlive dictionary usage.
workspaceSize: Use ZSTD_estimateDDictSize()
to determine how large workspace must be.
@return : pointer to ZSTD_DDict*, or NULL if error (size too small)
Note : there is no corresponding "free" function.
Since workspace was allocated externally, it must be freed externally.
</p></pre><BR>
<pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
</b><p> Provides the dictID stored within dictionary.
if @return == 0, the dictionary is not conformant with Zstandard specification.

5
lib/compress/zstd_compress.c
View File

@ -3237,10 +3237,11 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/*! ZSTD_estimateCDictSize() :
* Estimate amount of memory that will be needed to create a dictionary with following arguments */
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
{
cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams);
return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
+ (byReference ? 0 : dictSize);
}
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)

6
lib/decompress/zstd_decompress.c
View File

@ -2053,10 +2053,10 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
size_t ZSTD_estimateDDictSize(size_t dictSize)
* Note : dictionary created "byReference" are smaller */
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference)
{
return dictSize + sizeof(ZSTD_DDict);
return sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)

26
lib/zstd.h
View File

@ -345,8 +345,6 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#endif /* ZSTD_H_235446 */
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
/****************************************************************************************
* START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
@ -356,6 +354,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
* Use them only in association with static linking.
* ***************************************************************************************/
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
/* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
@ -500,9 +501,9 @@ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
/*! ZSTD_estimate?DictSize() :
* Note : if dictionary is created "byReference", reduce estimation by dictSize */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
* Note : dictionary created "byReference" are smaller */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
/***************************************
@ -842,6 +843,21 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
/*! ZSTD_initStaticDDict() :
* Generate a digested dictionary in provided memory area.
* workspace: The memory area to emplace the dictionary into.
* Provided pointer must 8-bytes aligned.
* It must outlive dictionary usage.
* workspaceSize: Use ZSTD_estimateDDictSize()
* to determine how large workspace must be.
* @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
* Note : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally.
*/
ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference);
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.

7
tests/fuzzer.c
View File

@ -490,14 +490,15 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1);
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
}
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem customMem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
ZSTD_customMem const customMem = { NULL, NULL, NULL };
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
1 /* by Referece */, cParams, customMem);
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict);
ZSTD_freeCDict(cdict);

6
tests/zstreamtest.c
View File

@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
size_t const s = ZSTD_estimateCStreamSize(cParams)
+ ZSTD_estimateCDictSize(cParams, dictSize); /* uses ZSTD_initCStream_usingDict() */
/* uses ZSTD_initCStream_usingDict() */
+ ZSTD_estimateCDictSize(cParams, dictSize, 0);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
}
@ -281,7 +282,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (gfhError!=0) goto _output_error;
DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
{ size_t const s = ZSTD_estimateDStreamSize(fhi)
+ ZSTD_estimateDDictSize(dictSize); /* uses ZSTD_initDStream_usingDict() */
/* uses ZSTD_initDStream_usingDict() */
+ ZSTD_estimateDDictSize(dictSize, 0);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
} }