diff --git a/NEWS b/NEWS index 50b26fb9..939c1ef9 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.4.4 Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode new : windows DLL project, thanks to Christophe Chevalier v0.4.3 : diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index 844c5040..c726f0de 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -160,6 +160,12 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) } +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* zbc, const void* src, size_t srcSize) +{ + ZSTD_compress_insertDictionary(zbc->zc, src, srcSize); + return 0; +} + /* *** Compression *** */ @@ -327,6 +333,8 @@ struct ZBUFF_DCtx_s { size_t outStart; size_t outEnd; size_t hPos; + const char* dict; + size_t dictSize; ZBUFF_dStage stage; unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -353,17 +361,23 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) } - /* *** Initialization *** */ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) { zbc->stage = ZBUFFds_readHeader; - zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0; return ZSTD_resetDCtx(zbc->zc); } +size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize) +{ + zbc->dict = src; + zbc->dictSize = srcSize; + return 0; +} + /* *** Decompression *** */ @@ -442,6 +456,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt if (zbc->outBuff == NULL) return ERROR(memory_allocation); } } + if (zbc->dictSize) + ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize); if (zbc->hPos) { /* some data already loaded into headerBuffer : transfer into inBuff */ diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 755728b1..d2316a82 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -69,6 +69,7 @@ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize); ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); @@ -81,6 +82,9 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDst * Use ZBUFF_compressInit() to start a new compression operation. * ZBUFF_CCtx objects can be reused multiple times. * +* Optionally, a reference to a static dictionary can be created with ZBUFF_compressWithDictionary() +* Note that the dictionary content must remain accessible during the compression process. +* * Use ZBUFF_compressContinue() repetitively to consume input stream. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr. @@ -115,6 +119,8 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize); + ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); /** ************************************************ @@ -125,6 +131,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* * Use ZBUFF_decompressInit() to start a new decompression operation. * ZBUFF_DCtx objects can be reused multiple times. * +* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() +* It must be the same content as the one set during compression phase. +* Dictionary content must remain accessible during the decompression process. +* * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 59039356..ab8aa348 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -161,6 +161,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZBUFF_compressInit(zc, 1); readSize = CNBufferSize; genSize = compressedBufferSize; + ZBUFF_compressWithDictionary(zc, CNBuffer, 128 KB); result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ @@ -174,6 +175,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInit(zd); + ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB); readSize = cSize; genSize = CNBufferSize; result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); @@ -244,7 +246,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit U32 coreSeed = seed, lseed = 0; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; - XXH64_state_t crc64; U32 startTime = FUZ_GetMilliStart(); /* allocation */ @@ -280,10 +281,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart; - size_t cSize; + const BYTE* dict; + size_t cSize, dictSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; + XXH64_state_t crc64; U64 crcOrig, crcDest; /* init */ @@ -316,6 +319,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); + + sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; + sampleSize = (size_t)1 << sampleSizeLog; + sampleSize += FUZ_rand(&lseed) & (sampleSize-1); + sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); + dict = srcBuffer + sampleStart; + dictSize = sampleSize; + ZBUFF_compressWithDictionary(zc, dict, dictSize); + totalTestSize = 0; cSize = 0; for (n=0; n