AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

first sketch for ZSTD_initCStream_usingCDict()

Browse Source
This commit is contained in:
Yann Collet 2016-10-25 16:19:52 -07:00
parent 62d9a7ddfd
commit 9516234e67
4 changed files with 52 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,6 +1,7 @@
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
New : doc/zstd_manual.html, by Przemyslaw Skibinski
Improved : slightly better compression ratio at --ultra levels (>= 20)
Changed : zstd_errors.h is now part of include installation
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets

30
lib/compress/zstd_compress.c
View File

@ -122,6 +122,11 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface *
return &(ctx->seqStore);
}
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
{
return cctx->params;
}
/** ZSTD_checkParams() :
ensure param values remain within authorized range.
@ -2779,6 +2784,10 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
}
}
static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
return ZSTD_getParamsFromCCtx(cdict->refContext);
}
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
@ -2815,7 +2824,8 @@ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s {
ZSTD_CCtx* cctx;
ZSTD_CDict* cdict;
ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@ -2858,7 +2868,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->cctx);
ZSTD_freeCDict(zcs->cdict);
ZSTD_freeCDict(zcs->cdictLocal);
ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem);
@ -2906,15 +2916,25 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
ZSTD_freeCDict(zcs->cdict);
zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
if (zcs->cdict == NULL) return ERROR(memory_allocation);
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal;
zcs->checksum = params.fParams.checksumFlag > 0;
return ZSTD_resetCStream(zcs, pledgedSrcSize);
}
/* note : cdict must outlive compression session */
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
{
ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
zcs->cdict = cdict;
return initError;
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);

2
lib/zstd.h
View File

@ -459,6 +459,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
@ -468,6 +469,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);

26
tests/zstreamtest.c
View File

@ -77,7 +77,7 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart)
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
@ -281,6 +281,28 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
}
DISPLAYLEVEL(4, "OK \n");
/* CDict scenario */
DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1);
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
if (ZSTD_isError(initError)) goto _output_error;
cSize = 0;
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
cSize = outBuff.pos;
ZSTD_freeCDict(cdict);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
}
DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_CStream(zc);
if (ZSTD_isError(s)) goto _output_error;
@ -511,7 +533,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
/* multi - fragments decompression test */
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
CHECK( ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
} else
ZSTD_initDStream_usingDict(zd, dict, dictSize);
{ size_t decompressionResult = 1;