AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #950 from facebook/srcSizeAdaptation

Browse Source 
fix adaptation on srcSize
This commit is contained in:
Yann Collet 2017-12-14 14:48:31 -08:00 committed by GitHub
commit a0ac8c895c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/compress/zstd_compress.c
View File

@ -144,6 +144,8 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor
static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize)
{
DEBUGLOG(4, "ZSTD_getCParamsFromCCtxParams: srcSize = %u, dictSize = %u",
(U32)srcSizeHint, (U32)dictSize);
return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ?
CCtxParams.cParams :
ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize);
@ -151,18 +153,22 @@ static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize)
{
DEBUGLOG(4, "ZSTD_cLevelToCCtxParams_srcSize: srcSize = %u",
(U32)srcSize);
CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0);
CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
}
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{
DEBUGLOG(4, "ZSTD_cLevelToCParams");
ZSTD_cLevelToCCtxParams_srcSize(
&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1);
}
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams)
{
DEBUGLOG(4, "ZSTD_cLevelToCCtxParams");
ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, 0);
}
@ -261,7 +267,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
case ZSTD_p_targetLength:
case ZSTD_p_compressionStrategy:
if (cctx->cdict) return ERROR(stage_wrong);
ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */
if (value>0) ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_contentSizeFlag:
@ -422,7 +428,7 @@ size_t ZSTD_CCtxParam_setParameter(
#endif
case ZSTD_p_enableLongDistanceMatching :
if (value != 0) {
if (value) {
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
}
@ -1736,6 +1742,8 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
op += cSize;
assert(dstCapacity >= cSize);
dstCapacity -= cSize;
DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u",
(U32)cSize);
} }
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
@ -3070,6 +3078,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
}
#endif
DEBUGLOG(4, "ZSTD_getCParams: cLevel=%i, srcSize=%u, dictSize=%u => table %u",
compressionLevel, (U32)srcSizeHint, (U32)dictSize, tableID);
if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
{ ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];

6
programs/fileio.c
View File

@ -454,7 +454,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, using good compression parameters */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, for compression parameters adaptation */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
@ -789,6 +789,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
&outBuff, &inBuff, ZSTD_e_continue) );
/* Write compressed stream */
DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_continue: generated %u bytes \n",
(U32)outBuff.pos);
if (outBuff.pos) {
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos)
@ -824,6 +826,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
EXM_THROW(26, "Compression error during frame end : %s",
ZSTD_getErrorName(result));
}
DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_end: generated %u bytes \n",
(U32)outBuff.pos);
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck != outBuff.pos)
EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);