AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove ZSTD_p_refDictContent and dictContentByRef

Browse Source
This commit is contained in:
Stella Lau 2017-08-25 10:48:07 -07:00
parent 15fdeb9e41
commit eb7bbab36a
5 changed files with 31 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/common/zstd_internal.h
View File

@ -294,7 +294,6 @@ struct ZSTD_CCtx_params_s {
U32 forceWindow; /* force back-references to respect limit of
* 1<<wLog, even for dictionary */
/* Dictionary */
U32 dictContentByRef;
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent"
* or "fullDict" only */

26
lib/compress/zstd_compress.c
View File

@ -336,7 +336,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_dictMode:
case ZSTD_p_refDictContent:
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@ -463,11 +462,6 @@ size_t ZSTD_CCtxParam_setParameter(
params->dictMode = (ZSTD_dictMode_e)value;
return 0;
case ZSTD_p_refDictContent :
/* dictionary content will be referenced, instead of copied */
params->dictContentByRef = value > 0;
return 0;
case ZSTD_p_forceMaxWindow :
params->forceWindow = value > 0;
return 0;
@ -514,7 +508,6 @@ size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params
/* Assume dictionary parameters are validated */
cctx->requestedParams.dictMode = params->dictMode;
cctx->requestedParams.dictContentByRef = params->dictContentByRef;
/* Set force window explicitly since it sets cctx->loadedDictEnd */
CHECK_F( ZSTD_CCtx_setParameter(
@ -541,7 +534,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
return 0;
}
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
size_t ZSTD_CCtx_loadDictionary_internal(
ZSTD_CCtx* cctx, const void* dict, size_t dictSize, unsigned byReference)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */
@ -557,7 +551,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
ZSTD_getCParams(cctx->requestedParams.compressionLevel, 0, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize,
cctx->requestedParams.dictContentByRef,
byReference,
cctx->requestedParams.dictMode,
cParams, cctx->customMem);
cctx->cdict = cctx->cdictLocal;
@ -567,6 +561,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
return 0;
}
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(
ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
{
return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, 1);
}
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
{
return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, 0);
}
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
@ -3896,7 +3902,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize,
params.dictContentByRef, params.dictMode,
0 /* byReference */, params.dictMode,
params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);

2
lib/compress/zstdmt_compress.c
View File

@ -344,7 +344,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
{ ZSTD_CCtx_params jobParams = job->params;
/* Force loading dictionary in "content-only" mode (no header analysis) */
size_t const dictModeError =
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_dictMode, 1);
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_dictMode, (U32)ZSTD_dm_rawContent);
size_t const forceWindowError =
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
/* Note: ZSTD_setCCtxParameter() should not be used here.

13
lib/zstd.h
View File

@ -970,8 +970,6 @@ typedef enum {
/* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
* It requires that dictionary buffer outlives its users */
/* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
@ -1015,8 +1013,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
* meaning "return to no-dictionary mode".
* Note 1 : `dict` content will be copied internally,
* except if ZSTD_p_refDictContent is set before loading.
* Note 1 : `dict` content will be copied internally. Use
* ZSTD_CCtx_loadDictionary_byReference() to reference dictionary
* content instead.
* Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
* For this reason, compression parameters cannot be changed anymore after loading a dictionary.
* It's also a CPU-heavy operation, with non-negligible impact on latency.
@ -1024,6 +1023,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
* To return to "no-dictionary" situation, load a NULL dictionary */
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_loadDictionary_byReference() :
* Same as ZSTD_CCtx_loadDictionary() except dictionary content will be
* referenced, instead of copied. The dictionary buffer must outlive its users.
*/
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() :
* Reference a prepared dictionary, to be used for all next compression jobs.
* Note that compression parameters are enforced from within CDict,

7
tests/zstreamtest.c
View File

@ -1366,7 +1366,6 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
/* multi-threading parameters */
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
@ -1386,7 +1385,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
}
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
} else {
CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
}
if (dict && dictSize) {
/* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
if (useOpaqueAPI) {