AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

extended static dictionary to buffered mode

Browse Source
This commit is contained in:
Yann Collet 2015-12-12 11:17:42 +01:00
parent 6c3e2e7855
commit 3a061a4c9f
4 changed files with 44 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,5 +1,6 @@
v0.4.4
Fixed : high compression modes for Windows 32 bits
new : external dictionary API extended to buffered mode
new : windows DLL project, thanks to Christophe Chevalier
v0.4.3 :

20
lib/zstd_buffered.c
View File

@ -160,6 +160,12 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
}
ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* zbc, const void* src, size_t srcSize)
{
ZSTD_compress_insertDictionary(zbc->zc, src, srcSize);
return 0;
}
/* *** Compression *** */
@ -327,6 +333,8 @@ struct ZBUFF_DCtx_s {
size_t outStart;
size_t outEnd;
size_t hPos;
const char* dict;
size_t dictSize;
ZBUFF_dStage stage;
unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@ -353,17 +361,23 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
}
/* *** Initialization *** */
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
{
zbc->stage = ZBUFFds_readHeader;
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0;
return ZSTD_resetDCtx(zbc->zc);
}
size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize)
{
zbc->dict = src;
zbc->dictSize = srcSize;
return 0;
}
/* *** Decompression *** */
@ -442,6 +456,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
}
if (zbc->dictSize)
ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize);
if (zbc->hPos)
{
/* some data already loaded into headerBuffer : transfer into inBuff */

10
lib/zstd_buffered.h
View File

@ -69,6 +69,7 @@ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
@ -81,6 +82,9 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDst
* Use ZBUFF_compressInit() to start a new compression operation.
* ZBUFF_CCtx objects can be reused multiple times.
*
* Optionally, a reference to a static dictionary can be created with ZBUFF_compressWithDictionary()
* Note that the dictionary content must remain accessible during the compression process.
*
* Use ZBUFF_compressContinue() repetitively to consume input stream.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
@ -115,6 +119,8 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
/** ************************************************
@ -125,6 +131,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t*
* Use ZBUFF_decompressInit() to start a new decompression operation.
* ZBUFF_DCtx objects can be reused multiple times.
*
* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary()
* It must be the same content as the one set during compression phase.
* Dictionary content must remain accessible during the decompression process.
*
* Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.

17
programs/zbufftest.c
View File

@ -161,6 +161,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZBUFF_compressInit(zc, 1);
readSize = CNBufferSize;
genSize = compressedBufferSize;
ZBUFF_compressWithDictionary(zc, CNBuffer, 128 KB);
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(result)) goto _output_error;
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
@ -174,6 +175,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInit(zd);
ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB);
readSize = cSize;
genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
@ -244,7 +246,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 coreSeed = seed, lseed = 0;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
XXH64_state_t crc64;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
@ -280,10 +281,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart;
size_t cSize;
const BYTE* dict;
size_t cSize, dictSize;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
XXH64_state_t crc64;
U64 crcOrig, crcDest;
/* init */
@ -316,6 +319,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
ZBUFF_compressWithDictionary(zc, dict, dictSize);
totalTestSize = 0;
cSize = 0;
for (n=0; n<nbChunks; n++)
@ -363,6 +375,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);
ZBUFF_decompressWithDictionary(zd, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)