AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

completed implementation of ZSTD_cParam_getBounds()

Browse Source 
for all parameters
This commit is contained in:
Yann Collet 2018-11-20 16:06:00 -08:00
parent 2e7fd6a2cb
commit 5c6d4b18ac
5 changed files with 99 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/zstd_manual.html
View File

@ -459,7 +459,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Special: value 0 means "use default value" (default: 64). */
ZSTD_p_ldmBucketSizeLog=163, </b>/* Log size of each bucket in the LDM hash table for collision resolution.<b>
* Larger values improve collision resolution but decrease compression speed.
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX .
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
* Special: value 0 means "use default value" (default: 3). */
ZSTD_p_ldmHashEveryLog=164, </b>/* Frequency of inserting/looking up entries into the LDM hash table.<b>
* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
@ -490,8 +490,13 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog=402, </b>/* Size of previous input reloaded at the beginning of each job, as a fraction of window size.<b>
* 0 : no overlap; 6(default) : use 1/8th of windowSize; >=9 : use full windowSize */
ZSTD_p_overlapSizeLog=402, </b>/* Size of previous job reloaded at the beginning of each job, as a fraction of window size.<b>
* This value is enforced only when nbWorkers >= 1.
* Larger values increase compression ratio, but decrease speed.
* Values range from 0 (no overlap) to 9 (overlap a full windowSize).
* Each rank (except 0) increase/decrease load size by a factor 2
* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:w/256;
* default value is 6 : use 1/8th of windowSize */
</b>/* =================================================================== */<b>
</b>/* experimental parameters - no stability guaranteed */<b>

84
lib/compress/zstd_compress.c
View File

@ -269,7 +269,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
case ZSTD_p_compressionStrategy:
bounds.lowerBound = (int)ZSTD_fast;
bounds.upperBound = (int)ZSTD_btultra;
bounds.upperBound = (int)ZSTD_btultra; /* note : how to ensure at compile time that this is the highest value strategy ? */
return bounds;
case ZSTD_p_contentSizeFlag:
@ -297,21 +297,65 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
return bounds;
case ZSTD_p_jobSize:
bounds.lowerBound = 0;
#ifdef ZSTD_MULTITHREAD
bounds.upperBound = ZSTDMT_JOBSIZE_MAX;
#else
bounds.upperBound = 0;
#endif
case ZSTD_p_overlapSizeLog:
case ZSTD_p_rsyncable:
bounds.lowerBound = ZSTD_OVERLAPLOG_MIN;
bounds.upperBound = ZSTD_OVERLAPLOG_MAX;
return bounds;
case ZSTD_p_enableLongDistanceMatching:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_p_ldmHashLog:
bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN;
bounds.upperBound = ZSTD_LDM_HASHLOG_MAX;
return bounds;
case ZSTD_p_ldmMinMatch:
bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN;
bounds.upperBound = ZSTD_LDM_MINMATCH_MAX;
return bounds;
case ZSTD_p_ldmBucketSizeLog:
bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN;
bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX;
return bounds;
case ZSTD_p_ldmHashEveryLog:
bounds.lowerBound = ZSTD_LDM_HASHEVERYLOG_MIN;
bounds.upperBound = ZSTD_LDM_HASHEVERYLOG_MAX;
return bounds;
/* experimental parameters */
case ZSTD_p_rsyncable:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_p_forceMaxWindow :
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_p_format:
ZSTD_STATIC_ASSERT((int)ZSTD_f_zstd1 < (int)ZSTD_f_zstd1_magicless);
bounds.lowerBound = (int)ZSTD_f_zstd1;
bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
return bounds;
case ZSTD_p_forceAttachDict:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
default:
{ ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 };
return boundError;
@ -402,7 +446,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_nbWorkers:
if ((value>0) && cctx->staticSize) {
if ((value!=0) && cctx->staticSize) {
return ERROR(parameter_unsupported); /* MT not compatible with static alloc */
}
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@ -447,31 +491,31 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
}
case ZSTD_p_windowLog :
if (value>0) /* 0 => use default */
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CCtxParams->cParams.windowLog = value;
return CCtxParams->cParams.windowLog;
case ZSTD_p_hashLog :
if (value>0) /* 0 => use default */
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CCtxParams->cParams.hashLog = value;
return CCtxParams->cParams.hashLog;
case ZSTD_p_chainLog :
if (value>0) /* 0 => use default */
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CCtxParams->cParams.chainLog = value;
return CCtxParams->cParams.chainLog;
case ZSTD_p_searchLog :
if (value>0) /* 0 => use default */
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CCtxParams->cParams.searchLog = value;
return value;
case ZSTD_p_minMatch :
if (value>0) /* 0 => use default */
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
CCtxParams->cParams.minMatch = value;
return CCtxParams->cParams.minMatch;
@ -482,29 +526,29 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
return CCtxParams->cParams.targetLength;
case ZSTD_p_compressionStrategy :
if (value>0) /* 0 => use default */
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, (int)ZSTD_fast, (int)ZSTD_btultra);
CCtxParams->cParams.strategy = (ZSTD_strategy)value;
return (size_t)CCtxParams->cParams.strategy;
case ZSTD_p_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */
DEBUGLOG(4, "set content size flag = %u", (value>0));
CCtxParams->fParams.contentSizeFlag = value > 0;
DEBUGLOG(4, "set content size flag = %u", (value!=0));
CCtxParams->fParams.contentSizeFlag = value != 0;
return CCtxParams->fParams.contentSizeFlag;
case ZSTD_p_checksumFlag :
/* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
CCtxParams->fParams.checksumFlag = value > 0;
CCtxParams->fParams.checksumFlag = value != 0;
return CCtxParams->fParams.checksumFlag;
case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
CCtxParams->fParams.noDictIDFlag = !value;
return !CCtxParams->fParams.noDictIDFlag;
case ZSTD_p_forceMaxWindow :
CCtxParams->forceWindow = (value > 0);
CCtxParams->forceWindow = (value != 0);
return CCtxParams->forceWindow;
case ZSTD_p_forceAttachDict : {
@ -516,7 +560,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_p_nbWorkers :
#ifndef ZSTD_MULTITHREAD
if (value>0) return ERROR(parameter_unsupported);
if (value!=0) return ERROR(parameter_unsupported);
return 0;
#else
return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value);
@ -544,24 +588,24 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
#endif
case ZSTD_p_enableLongDistanceMatching :
CCtxParams->ldmParams.enableLdm = (value>0);
CCtxParams->ldmParams.enableLdm = (value!=0);
return CCtxParams->ldmParams.enableLdm;
case ZSTD_p_ldmHashLog :
if (value>0) /* 0 ==> auto */
if (value!=0) /* 0 ==> auto */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CCtxParams->ldmParams.hashLog = value;
return CCtxParams->ldmParams.hashLog;
case ZSTD_p_ldmMinMatch :
if (value>0) /* 0 ==> default */
if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
CCtxParams->ldmParams.minMatchLength = value;
return CCtxParams->ldmParams.minMatchLength;
case ZSTD_p_ldmBucketSizeLog :
if (value > ZSTD_LDM_BUCKETSIZELOG_MAX)
return ERROR(parameter_outOfBound);
if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX);
CCtxParams->ldmParams.bucketSizeLog = value;
return CCtxParams->ldmParams.bucketSizeLog;

1
lib/compress/zstdmt_compress.c
View File

@ -10,7 +10,6 @@
/* ====== Tuning parameters ====== */
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */
#define ZSTDMT_OVERLAPLOG_DEFAULT 6

8
lib/compress/zstdmt_compress.h
View File

@ -30,6 +30,10 @@
/* === Constants === */
#define ZSTDMT_NBWORKERS_MAX 200
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1 GB)) /* note : limited by `jobSize` type, which is `int` */
/* === Memory management === */
@ -64,10 +68,6 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
/* === Advanced functions and parameters === */
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,

31
lib/zstd.h
View File

@ -553,7 +553,7 @@ typedef enum {
* Special: value 0 means "use default value" (default: 64). */
ZSTD_p_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
* Larger values improve collision resolution but decrease compression speed.
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX .
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
* Special: value 0 means "use default value" (default: 3). */
ZSTD_p_ldmHashEveryLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
@ -584,8 +584,13 @@ typedef enum {
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog=402, /* Size of previous input reloaded at the beginning of each job, as a fraction of window size.
* 0 : no overlap; 6(default) : use 1/8th of windowSize; >=9 : use full windowSize */
ZSTD_p_overlapSizeLog=402, /* Size of previous job reloaded at the beginning of each job, as a fraction of window size.
* This value is enforced only when nbWorkers >= 1.
* Larger values increase compression ratio, but decrease speed.
* Values range from 0 (no overlap) to 9 (overlap a full windowSize).
* Each rank (except 0) increase/decrease load size by a factor 2
* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:w/256;
* default value is 6 : use 1/8th of windowSize */
/* =================================================================== */
/* experimental parameters - no stability guaranteed */
@ -886,8 +891,6 @@ ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
#define ZSTD_SKIPPABLEHEADERSIZE 8
#define ZSTD_HASHLOG3_MAX 17
/* note : matches --ultra and --long default (27) ? */
#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
* requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size, to preserve memory.
@ -912,10 +915,22 @@ ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
#define ZSTD_OVERLAPLOG_MIN 0
#define ZSTD_OVERLAPLOG_MAX 9
/* LDM parameter bounds */
#define ZSTD_LDM_MINMATCH_MAX 4096
#define ZSTD_LDM_MINMATCH_MIN 4
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
#define ZSTD_LDM_MINMATCH_MIN 4
#define ZSTD_LDM_MINMATCH_MAX 4096
#define ZSTD_LDM_BUCKETSIZELOG_MIN 1
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
#define ZSTD_LDM_HASHEVERYLOG_MIN 0
#define ZSTD_LDM_HASHEVERYLOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
/* internal */
#define ZSTD_HASHLOG3_MAX 17
/* --- Advanced types --- */