diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 8cac4294..d7091545 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -372,7 +372,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. `frameContentSize` is an optional parameter, provide `0` if unknown
@@ -382,7 +382,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -Gives the amount of memory used by a given ZSTD_CCtx +
amount of used memory is variable, depending primarily on compression level
typedef enum { @@ -507,7 +507,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; vAdvanced streaming functions
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);/**< size of CStream is variable, depending primarily on compression level */ +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< same as ZSTD_sizeof_CCtx */ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d55ea505..1248032b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -244,7 +244,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u } -size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming) { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.searchLength==3) ? 3 : 4; @@ -260,12 +260,17 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) + entropyScratchSpace_size; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); /* allocate buffers */ - { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + { size_t const neededInBuffSize = ((size_t)1 << params.cParams.windowLog) + zcs->blockSize; if (zcs->inBuffSize < neededInBuffSize) { zcs->inBuffSize = 0; ZSTD_free(zcs->inBuff, zcs->customMem); - zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); + zcs->inBuff = (char*)ZSTD_malloc(neededInBuffSize, zcs->customMem); if (zcs->inBuff == NULL) return ERROR(memory_allocation); zcs->inBuffSize = neededInBuffSize; } - zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); } if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { size_t const outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; zcs->outBuffSize = 0; ZSTD_free(zcs->outBuff, zcs->customMem); - zcs->outBuff = (char*) ZSTD_malloc(outBuffSize, zcs->customMem); + zcs->outBuff = (char*)ZSTD_malloc(outBuffSize, zcs->customMem); if (zcs->outBuff == NULL) return ERROR(memory_allocation); zcs->outBuffSize = outBuffSize; } diff --git a/lib/zstd.h b/lib/zstd.h index bb447ee1..e8614334 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -460,8 +460,10 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t ***************************************/ /*! ZSTD_estimateCCtxSize() : * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. - * `frameContentSize` is an optional parameter, provide `0` if unknown */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); + * Set streaming to 1 if the CCtx will be used for streaming (CStream) + * Note : this function is currently unable to estimate additional memory allocation needed to create an internal CDict + * which can only happen when starting with ZSTD_initCStream_usingDict() */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ @@ -599,7 +601,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< same as ZSTD_sizeof_CCtx */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ -ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */ +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 1913b54d..3cc916e0 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -388,8 +388,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, 0/*streaming*/); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, 0); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);