CLI : automatically set overlap size to max (windowSize) for max compression level
This commit is contained in:
parent
06e7697f96
commit
8dafb1acf5
@ -285,6 +285,7 @@ struct ZSTDMT_CCtx_s {
|
||||
unsigned nextJobID;
|
||||
unsigned frameEnded;
|
||||
unsigned allJobsCompleted;
|
||||
unsigned overlapWrLog;
|
||||
unsigned long long frameContentSize;
|
||||
size_t sectionSize;
|
||||
ZSTD_CDict* cdict;
|
||||
@ -298,7 +299,6 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
|
||||
U32 const minNbJobs = nbThreads + 2;
|
||||
U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
|
||||
U32 const nbJobs = 1 << nbJobsLog2;
|
||||
//nbThreads = 1; /* for tests */
|
||||
DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n",
|
||||
nbThreads, minNbJobs, nbJobsLog2, nbJobs);
|
||||
if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
|
||||
@ -308,6 +308,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
|
||||
cctx->jobIDMask = nbJobs - 1;
|
||||
cctx->allJobsCompleted = 1;
|
||||
cctx->sectionSize = 0;
|
||||
cctx->overlapWrLog = 3;
|
||||
cctx->factory = POOL_create(nbThreads, 1);
|
||||
cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
|
||||
cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
|
||||
@ -367,6 +368,9 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
|
||||
case ZSTDMT_p_sectionSize :
|
||||
mtctx->sectionSize = value;
|
||||
return 0;
|
||||
case ZSTDMT_p_overlapSectionRLog :
|
||||
mtctx->overlapWrLog = value;
|
||||
return 0;
|
||||
default :
|
||||
return ERROR(compressionParameter_unsupported);
|
||||
}
|
||||
@ -510,9 +514,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
|
||||
zcs->frameContentSize = pledgedSrcSize;
|
||||
zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
|
||||
zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
|
||||
//zcs->targetDictSize = ((size_t)1 << zcs->params.cParams.windowLog); /* full window size, for test */
|
||||
zcs->targetDictSize = ((size_t)1 << zcs->params.cParams.windowLog) >> 3; /* fixed currently */
|
||||
//zcs->targetDictSize = 0;
|
||||
zcs->targetDictSize = zcs->overlapWrLog < 10 ? (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapWrLog) : 0;
|
||||
zcs->inBuffSize = zcs->targetSectionSize + ((size_t)1 << zcs->params.cParams.windowLog) /* margin */ + zcs->targetDictSize;
|
||||
zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
|
||||
if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
|
||||
|
@ -52,7 +52,8 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* d
|
||||
/* ZSDTMT_parameter :
|
||||
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
|
||||
typedef enum {
|
||||
ZSTDMT_p_sectionSize /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
|
||||
ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
|
||||
ZSTDMT_p_overlapSectionRLog /* reverse log of overlapped section; 0 == use a complete window, 3(default) == use 1/8th of window, values >=10 means no overlap */
|
||||
} ZSDTMT_parameter;
|
||||
|
||||
/* ZSTDMT_setMTCtxParameter() :
|
||||
|
@ -7,6 +7,7 @@
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Compiler Options
|
||||
***************************************/
|
||||
@ -266,10 +267,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
|
||||
if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
|
||||
if (cLevel==ZSTD_maxCLevel())
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionRLog, 0); /* use complete window for overlap */
|
||||
#else
|
||||
ress.cctx = ZSTD_createCStream();
|
||||
#endif
|
||||
if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
|
||||
#endif
|
||||
ress.srcBufferSize = ZSTD_CStreamInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZSTD_CStreamOutSize();
|
||||
|
@ -12,12 +12,13 @@
|
||||
#define FILEIO_H_23981798732
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||
#include "zstd.h" /* ZSTD_compressionParameters */
|
||||
#include "zstd.h" /* ZSTD_* */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Special i/o constants
|
||||
**************************************/
|
||||
|
@ -20,6 +20,7 @@
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Dependencies
|
||||
**************************************/
|
||||
|
@ -992,8 +992,8 @@ int main(int argc, const char** argv)
|
||||
int mainPause = 0;
|
||||
int mtOnly = 0;
|
||||
const char* const programName = argv[0];
|
||||
ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
|
||||
ZSTD_customMem customNULL = { NULL, NULL, NULL };
|
||||
ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
|
||||
ZSTD_customMem const customNULL = { NULL, NULL, NULL };
|
||||
|
||||
/* Check command line */
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
|
Loading…
Reference in New Issue
Block a user