AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added ZSTD_estimateCCtxSize()

Browse Source
This commit is contained in:
Yann Collet 2016-07-11 03:12:17 +02:00
parent 677ed26aa7
commit 3ae543ce75
4 changed files with 37 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/common/zstd.h
View File

@ -61,7 +61,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0
#define ZSTD_VERSION_MINOR 7
#define ZSTD_VERSION_RELEASE 3
#define ZSTD_VERSION_RELEASE 4
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@ -262,6 +262,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
/*-*************************************
* Advanced compression functions
***************************************/
/*! ZSTD_estimateCCtxSize() :
* Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
* `frameContentSize` is an optional parameter, provide `0` if unknown */
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize);
/*! ZSTD_createCCtx_advanced() :
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);

35
lib/compress/zstd_compress.c
View File

@ -175,6 +175,11 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
return 0; /* reserved as a potential error code in the future */
}
size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx)
{
return sizeof(*cctx) + cctx->workSpaceSize;
}
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */
{
return &(ctx->seqStore);
@ -244,17 +249,27 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
}
size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface, for paramagrill */
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize)
{
ZSTD_CCtx* const zc = ZSTD_createCCtx();
ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = cParams;
params.fParams.contentSizeFlag = 1;
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
{ size_t const ccsize = sizeof(*zc) + zc->workSpaceSize;
ZSTD_freeCCtx(zc);
return ccsize; }
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
const U32 divider = (cParams.searchLength==3) ? 3 : 4;
const size_t maxNbSeq = blockSize / divider;
const size_t tokenSpace = blockSize + 11*maxNbSeq;
const size_t chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog);
const size_t hSize = ((size_t)1) << cParams.hashLog;
const U32 hashLog3 = (cParams.searchLength>3) ? 0 :
( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX :
((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) );
const size_t h3Size = ((size_t)1) << hashLog3;
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((cParams.strategy == ZSTD_btopt) ? optSpace : 0);
return sizeof(ZSTD_CCtx) + neededSpace;
}
/*! ZSTD_resetCCtx_advanced() :

4
lib/decompress/zstd_decompress.c
View File

@ -949,7 +949,7 @@ ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, siz
}
size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length)
size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
{
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
memset(dst, byte, length);
@ -1001,7 +1001,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
break;
case bt_rle :
decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize);
decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
break;
case bt_end :
/* end of frame */

10
programs/paramgrill.c
View File

@ -48,7 +48,7 @@
#endif
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */
#include "zstd.h"
#include "datagen.h"
#include "xxhash.h"
@ -274,6 +274,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
const int startTime =BMK_GetMilliStart();
DISPLAY("\r%79s\r", "");
memset(&params, 0, sizeof(params));
params.cParams = cParams;
params.fParams.contentSizeFlag = 0;
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
@ -407,8 +408,6 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
BMK_printWinners2(stdout, winners, srcSize);
}
size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters params); /* hidden interface, declared here */
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx)
@ -442,8 +441,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_sizeofCCtx(params);
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_sizeofCCtx(winners[cLevel].params);
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, srcSize);
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, srcSize);
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
@ -453,7 +452,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log((double)testResult.dSpeed);
double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed);
if (W_DMemUsed_note < O_DMemUsed_note) {
/* uses too much Decompression memory for too little benefit */
if (W_ratio > O_ratio)