From eee789b7ea7b7841dd83067606007ebf410fe6df Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Dec 2018 17:41:42 -0800 Subject: [PATCH] continued: changed to overlapLog in deeper code layer. for consistency. --- lib/compress/zstd_compress.c | 6 +- lib/compress/zstd_compress_internal.h | 8 +-- lib/compress/zstdmt_compress.c | 92 ++++++++++++++++++--------- lib/compress/zstdmt_compress.h | 18 +++--- tests/zstreamtest.c | 2 +- 5 files changed, 79 insertions(+), 47 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8d61fdbd..d83d2e78 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -591,7 +591,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value); + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value); #endif case ZSTD_c_rsyncable : @@ -702,7 +702,7 @@ size_t ZSTD_CCtxParam_getParameter( #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - *value = CCtxParams->overlapSizeLog; + *value = CCtxParams->overlapLog; break; #endif case ZSTD_c_rsyncable : @@ -873,7 +873,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams) CLAMP(ZSTD_c_searchLog, cParams.searchLog); CLAMP(ZSTD_c_minMatch, cParams.minMatch); CLAMP(ZSTD_c_targetLength,cParams.targetLength); - CLAMP_TYPE(ZSTD_c_strategy, cParams.strategy, ZSTD_strategy); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); return cParams; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 680bd914..ce012b0f 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -190,10 +190,10 @@ struct ZSTD_CCtx_params_s { ZSTD_dictAttachPref_e attachDictPref; /* Multithreading: used to pass parameters to mtctx */ - unsigned nbWorkers; - unsigned jobSize; - unsigned overlapSizeLog; - unsigned rsyncable; + int nbWorkers; + size_t jobSize; + int overlapLog; + int rsyncable; /* Long distance matching parameters */ ldmParams_t ldmParams; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 4ce3dda0..468c4f68 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -10,7 +10,7 @@ /* ====== Tuning parameters ====== */ -#define ZSTDMT_OVERLAPLOG_DEFAULT 6 +#define ZSTDMT_OVERLAPLOG_DEFAULT 0 /* ====== Compiler specifics ====== */ @@ -340,8 +340,8 @@ static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) typedef struct { ZSTD_pthread_mutex_t poolMutex; - unsigned totalCCtx; - unsigned availCCtx; + int totalCCtx; + int availCCtx; ZSTD_customMem cMem; ZSTD_CCtx* cctx[1]; /* variable size */ } ZSTDMT_CCtxPool; @@ -349,16 +349,16 @@ typedef struct { /* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) { - unsigned u; - for (u=0; utotalCCtx; u++) - ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */ + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ ZSTD_pthread_mutex_destroy(&pool->poolMutex); ZSTD_free(pool, pool->cMem); } /* ZSTDMT_createCCtxPool() : * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ -static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers, +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, ZSTD_customMem cMem) { ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( @@ -379,7 +379,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers, } static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, - unsigned nbWorkers) + int nbWorkers) { if (srcPool==NULL) return NULL; if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ @@ -866,7 +866,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker { if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX; params->nbWorkers = nbWorkers; - params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT; + params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT; params->jobSize = 0; return nbWorkers; } @@ -976,28 +976,36 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) } /* Internal only */ -size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, - ZSTDMT_parameter parameter, unsigned value) { +size_t +ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, + ZSTDMT_parameter parameter, + int value) +{ DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter"); switch(parameter) { case ZSTDMT_p_jobSize : - DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value); - if ( (value > 0) /* value==0 => automatic job size */ - & (value < ZSTDMT_JOBSIZE_MIN) ) - value = ZSTDMT_JOBSIZE_MIN; - if (value > ZSTDMT_JOBSIZE_MAX) - value = ZSTDMT_JOBSIZE_MAX; - params->jobSize = value; - return value; - case ZSTDMT_p_overlapSectionLog : - if (value > 9) value = 9; - DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value); - params->overlapSizeLog = (value >= 9) ? 9 : value; + DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value); + if (value < ZSTDMT_JOBSIZE_MIN) value = ZSTDMT_JOBSIZE_MIN; + assert(value >= 0); + { size_t jobSize = value; + if (jobSize > ZSTDMT_JOBSIZE_MAX) jobSize = ZSTDMT_JOBSIZE_MAX; + params->jobSize = jobSize; + return jobSize; + } + + case ZSTDMT_p_overlapLog : + DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %i", value); + if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN; + if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX; + params->overlapLog = value; return value; + case ZSTDMT_p_rsyncable : - params->rsyncable = (value == 0 ? 0 : 1); + value = (value != 0); + params->rsyncable = value; return value; + default : return ERROR(parameter_unsupported); } @@ -1015,8 +1023,8 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, case ZSTDMT_p_jobSize: *value = mtctx->params.jobSize; break; - case ZSTDMT_p_overlapSectionLog: - *value = mtctx->params.overlapSizeLog; + case ZSTDMT_p_overlapLog: + *value = mtctx->params.overlapLog; break; case ZSTDMT_p_rsyncable: *value = mtctx->params.rsyncable; @@ -1154,7 +1162,7 @@ static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params) { - unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog; + unsigned const overlapRLog = (params.overlapLog>9) ? 0 : 9-params.overlapLog; if (params.ldmParams.enableLdm) return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog); return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog); @@ -1303,21 +1311,43 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, ZSTD_CCtx_params cctxParams = mtctx->params; cctxParams.cParams = params.cParams; cctxParams.fParams = params.fParams; - cctxParams.overlapSizeLog = overlapLog; + cctxParams.overlapLog = overlapLog; return ZSTDMT_compress_advanced_internal(mtctx, dst, dstCapacity, src, srcSize, cdict, cctxParams); } +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) +{ + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default: + return 6; + } + assert(0); +} + size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { - U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT; ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); + int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy); params.fParams.contentSizeFlag = 1; return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog); } @@ -1381,7 +1411,7 @@ size_t ZSTDMT_initCStream_internal( } mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params); - DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(mtctx->targetPrefixSize>>10)); + DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); mtctx->targetSectionSize = params.jobSize; if (mtctx->targetSectionSize == 0) { mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); @@ -1397,7 +1427,7 @@ size_t ZSTDMT_initCStream_internal( mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); } if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ - DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), params.jobSize); + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); { diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index c60ec832..61bbeb27 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -29,11 +29,13 @@ /* === Constants === */ -#define ZSTDMT_NBWORKERS_MAX 200 -#ifndef ZSTDMT_JOBSIZE_MIN -# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ +#ifndef ZSTDMT_NBWORKERS_MAX +# define ZSTDMT_NBWORKERS_MAX 200 #endif -#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1 GB)) /* note : limited by `jobSize` type, which is `int` */ +#ifndef ZSTDMT_JOBSIZE_MIN +# define ZSTDMT_JOBSIZE_MIN (1 MB) +#endif +#define ZSTDMT_JOBSIZE_MAX ((size_t)(MEM_32bits() ? (512 MB) : (1 GB))) /* === Memory management === */ @@ -89,9 +91,9 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, /* ZSTDMT_parameter : * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ typedef enum { - ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ - ZSTDMT_p_overlapSectionLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ - ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ + ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ + ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ + ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ } ZSTDMT_parameter; /* ZSTDMT_setMTCtxParameter() : @@ -135,7 +137,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); /*! ZSTDMT_CCtxParam_setMTCtxParameter() * like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */ -size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); +size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value); /*! ZSTDMT_CCtxParam_setNbWorkers() * Set nbWorkers, and clamp it. diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 0e23d9e2..db2182bf 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1547,7 +1547,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); + CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapLog, FUZ_rand(&lseed) % 12) ); CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */ CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); } } }