diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 988e133d..5f0bf2ab 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -285,6 +285,7 @@ struct ZSTDMT_CCtx_s { unsigned nextJobID; unsigned frameEnded; unsigned allJobsCompleted; + unsigned overlapWrLog; unsigned long long frameContentSize; size_t sectionSize; ZSTD_CDict* cdict; @@ -298,7 +299,6 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads) U32 const minNbJobs = nbThreads + 2; U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1; U32 const nbJobs = 1 << nbJobsLog2; - //nbThreads = 1; /* for tests */ DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n", nbThreads, minNbJobs, nbJobsLog2, nbJobs); if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL; @@ -308,6 +308,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads) cctx->jobIDMask = nbJobs - 1; cctx->allJobsCompleted = 1; cctx->sectionSize = 0; + cctx->overlapWrLog = 3; cctx->factory = POOL_create(nbThreads, 1); cctx->buffPool = ZSTDMT_createBufferPool(nbThreads); cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads); @@ -367,6 +368,9 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, case ZSTDMT_p_sectionSize : mtctx->sectionSize = value; return 0; + case ZSTDMT_p_overlapSectionRLog : + mtctx->overlapWrLog = value; + return 0; default : return ERROR(compressionParameter_unsupported); } @@ -510,9 +514,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, zcs->frameContentSize = pledgedSrcSize; zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2); zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize); - //zcs->targetDictSize = ((size_t)1 << zcs->params.cParams.windowLog); /* full window size, for test */ - zcs->targetDictSize = ((size_t)1 << zcs->params.cParams.windowLog) >> 3; /* fixed currently */ - //zcs->targetDictSize = 0; + zcs->targetDictSize = zcs->overlapWrLog < 10 ? (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapWrLog) : 0; zcs->inBuffSize = zcs->targetSectionSize + ((size_t)1 << zcs->params.cParams.windowLog) /* margin */ + zcs->targetDictSize; zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation); diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 4757e3e0..92de52d6 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -52,7 +52,8 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* d /* ZSDTMT_parameter : * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ typedef enum { - ZSTDMT_p_sectionSize /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */ + ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */ + ZSTDMT_p_overlapSectionRLog /* reverse log of overlapped section; 0 == use a complete window, 3(default) == use 1/8th of window, values >=10 means no overlap */ } ZSDTMT_parameter; /* ZSTDMT_setMTCtxParameter() : diff --git a/programs/fileio.c b/programs/fileio.c index f18e418a..ac7dffb3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -7,6 +7,7 @@ * of patent rights can be found in the PATENTS file in the same directory. */ + /* ************************************* * Compiler Options ***************************************/ @@ -266,10 +267,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, #ifdef ZSTD_MULTITHREAD ress.cctx = ZSTDMT_createCCtx(g_nbThreads); + if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); + if (cLevel==ZSTD_maxCLevel()) + ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionRLog, 0); /* use complete window for overlap */ #else ress.cctx = ZSTD_createCStream(); -#endif if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); +#endif ress.srcBufferSize = ZSTD_CStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZSTD_CStreamOutSize(); diff --git a/programs/fileio.h b/programs/fileio.h index 19f09c33..11178bcc 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -12,12 +12,13 @@ #define FILEIO_H_23981798732 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ -#include "zstd.h" /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_* */ #if defined (__cplusplus) extern "C" { #endif + /* ************************************* * Special i/o constants **************************************/ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 549dad01..64f2c919 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -20,6 +20,7 @@ #endif + /*-************************************ * Dependencies **************************************/ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 2cb6d65e..bef8734c 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -992,8 +992,8 @@ int main(int argc, const char** argv) int mainPause = 0; int mtOnly = 0; const char* const programName = argv[0]; - ZSTD_customMem customMem = { allocFunction, freeFunction, NULL }; - ZSTD_customMem customNULL = { NULL, NULL, NULL }; + ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL }; + ZSTD_customMem const customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb