From c7a18b7c219b10c1ca8b33cd0d3f2614cae7ff54 Mon Sep 17 00:00:00 2001 From: Stella Lau Date: Tue, 29 Aug 2017 15:10:42 -0700 Subject: [PATCH] Localize 'dictMode' from cctx to function param --- lib/common/zstd_internal.h | 4 +- lib/compress/zstd_compress.c | 104 ++++++++++++++++----------------- lib/compress/zstdmt_compress.c | 10 ++-- lib/zstd.h | 23 ++++---- 4 files changed, 69 insertions(+), 72 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index bfec42fa..6097f8e6 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -293,9 +293,6 @@ struct ZSTD_CCtx_params_s { int compressionLevel; U32 forceWindow; /* force back-references to respect limit of * 1<requestedParams, param, value); - case ZSTD_p_dictMode: - if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, * even when referencing into Dictionary content * default : 0 when using a CDict, 1 when using a Prefix */ @@ -447,15 +448,6 @@ size_t ZSTD_CCtxParam_setParameter( params->fParams.noDictIDFlag = (value == 0); return 0; - case ZSTD_p_dictMode : - /* restrict dictionary mode to "rawContent" or "fullDict" only */ - ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent); - if (value > (unsigned)ZSTD_dm_fullDict) { - return ERROR(parameter_outOfBound); - } - params->dictMode = (ZSTD_dictMode_e)value; - return 0; - case ZSTD_p_forceMaxWindow : params->forceWindow = value > 0; return 0; @@ -496,9 +488,6 @@ size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params cctx->requestedParams.fParams = params->fParams; cctx->requestedParams.compressionLevel = params->compressionLevel; - /* Assume dictionary parameters are validated */ - cctx->requestedParams.dictMode = params->dictMode; - /* Set force window explicitly since it sets cctx->loadedDictEnd */ CHECK_F( ZSTD_CCtx_setParameter( cctx, ZSTD_p_forceMaxWindow, params->forceWindow) ); @@ -524,9 +513,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo return 0; } -size_t ZSTD_CCtx_loadDictionary_internal( - ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod) +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ @@ -541,7 +530,7 @@ size_t ZSTD_CCtx_loadDictionary_internal( cctx->cdictLocal = ZSTD_createCDict_advanced( dict, dictSize, dictLoadMethod, - cctx->requestedParams.dictMode, + dictMode, cParams, cctx->customMem); cctx->cdict = cctx->cdictLocal; if (cctx->cdictLocal == NULL) @@ -553,12 +542,14 @@ size_t ZSTD_CCtx_loadDictionary_internal( ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( ZSTD_CCtx* cctx, const void* dict, size_t dictSize) { - return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, ZSTD_dlm_byRef); + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto); } ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) { - return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, ZSTD_dlm_byCopy); + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto); } @@ -566,20 +557,27 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); cctx->cdict = cdict; - cctx->prefix = NULL; /* exclusive */ - cctx->prefixSize = 0; + memset(&cctx->prefixDict, 0, sizeof(ZSTD_prefixDict)); /* exclusive */ return 0; } size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dm_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); cctx->cdict = NULL; /* prefix discards any prior cdict */ - cctx->prefix = prefix; - cctx->prefixSize = prefixSize; + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictMode = dictMode; return 0; } + static void ZSTD_startNewCompression(ZSTD_CCtx* cctx) { cctx->streamStage = zcss_init; @@ -3387,13 +3385,14 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { DEBUGLOG(4, "ZSTD_compressBegin_internal"); DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none")); - DEBUGLOG(4, "dictMode : %u", (U32)(params.dictMode)); + DEBUGLOG(4, "dictMode : %u", (U32)dictMode); /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ @@ -3406,18 +3405,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); - return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode); + return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode); } size_t ZSTD_compressBegin_advanced_internal( ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { /* compression parameters verification and optimization */ CHECK_F( ZSTD_checkCParams(params.cParams) ); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, + return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, params, pledgedSrcSize, ZSTDb_not_buffered); } @@ -3430,8 +3430,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, { ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); - cctxParams.dictMode = ZSTD_dm_auto; - return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, cctxParams, + return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dm_auto, + cctxParams, pledgedSrcSize); } @@ -3440,8 +3440,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); - cctxParams.dictMode = ZSTD_dm_auto; - return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, cctxParams, 0, ZSTDb_not_buffered); } @@ -3523,7 +3522,6 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, { ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); - cctxParams.dictMode = ZSTD_dm_auto; return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, @@ -3549,7 +3547,7 @@ size_t ZSTD_compress_advanced_internal( const void* dict,size_t dictSize, ZSTD_CCtx_params params) { - CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, + CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, params, srcSize, ZSTDb_not_buffered) ); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -3633,9 +3631,8 @@ static size_t ZSTD_initCDict_internal( ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams; cctxParams.fParams = fParams; cctxParams.cParams = cParams; - cctxParams.dictMode = dictMode; CHECK_F( ZSTD_compressBegin_internal(cdict->refContext, - cdict->dictContent, dictSize, + cdict->dictContent, dictSize, dictMode, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered) ); @@ -3759,10 +3756,9 @@ size_t ZSTD_compressBegin_usingCDict_advanced( { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ZSTD_getCParamsFromCDict(cdict); params.fParams = fParams; - params.dictMode = ZSTD_dm_auto; DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); return ZSTD_compressBegin_internal(cctx, - NULL, 0, + NULL, 0, ZSTD_dm_auto, cdict, params, pledgedSrcSize, ZSTDb_not_buffered); @@ -3841,7 +3837,7 @@ size_t ZSTD_CStreamOutSize(void) static size_t ZSTD_resetCStream_internal( ZSTD_CStream* zcs, - const void* dict, size_t dictSize, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, const ZSTD_CDict* cdict, const ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { @@ -3851,7 +3847,7 @@ static size_t ZSTD_resetCStream_internal( assert(!((dict) && (cdict))); /* either dict or cdict, not both */ CHECK_F( ZSTD_compressBegin_internal(zcs, - dict, dictSize, + dict, dictSize, dictMode, cdict, params, pledgedSrcSize, ZSTDb_buffered) ); @@ -3871,7 +3867,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) params.fParams.contentSizeFlag = (pledgedSrcSize > 0); params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0); DEBUGLOG(5, "ZSTD_resetCStream"); - return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize); + return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); } /*! ZSTD_initCStream_internal() : @@ -3895,7 +3891,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, ZSTD_freeCDict(zcs->cdictLocal); zcs->cdictLocal = ZSTD_createCDict_advanced( dict, dictSize, - ZSTD_dlm_byCopy, params.dictMode, + ZSTD_dlm_byCopy, ZSTD_dm_auto, params.cParams, zcs->customMem); zcs->cdict = zcs->cdictLocal; if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); @@ -3911,7 +3907,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, params.compressionLevel = ZSTD_CLEVEL_CUSTOM; zcs->requestedParams = params; - return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize); + return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); } /* ZSTD_initCStream_usingCDict_advanced() : @@ -4157,23 +4153,27 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { - const void* const prefix = cctx->prefix; - size_t const prefixSize = cctx->prefixSize; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ZSTD_getCParamsFromCCtxParams( cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); - cctx->prefix = NULL; cctx->prefixSize = 0; /* single usage */ - assert(prefix==NULL || cctx->cdict==NULL); /* only one can be set */ + memset(&cctx->prefixDict, 0, sizeof(ZSTD_prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ #ifdef ZSTD_MULTITHREAD if (params.nbThreads > 1) { DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads); - CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + CHECK_F( ZSTDMT_initCStream_internal( + cctx->mtctx, prefixDict.dict, prefixDict.dictSize, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); cctx->streamStage = zcss_load; } else #endif { - CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + CHECK_F( ZSTD_resetCStream_internal( + cctx, prefixDict.dict, prefixDict.dictSize, + prefixDict.dictMode, cctx->cdict, params, + cctx->pledgedSrcSizePlusOne-1) ); } } /* compression stage */ diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index e64a1f69..41278677 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -343,13 +343,13 @@ void ZSTDMT_compressChunk(void* jobDescription) if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ { ZSTD_CCtx_params jobParams = job->params; /* Force loading dictionary in "content-only" mode (no header analysis) */ - size_t const dictModeError = - ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_dictMode, (U32)ZSTD_dm_rawContent); size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); - size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, jobParams, job->fullFrameSize); - if (ZSTD_isError(initError) || ZSTD_isError(dictModeError) || - ZSTD_isError(forceWindowError)) { job->cSize = initError; goto _endJob; } + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize); + if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) { + job->cSize = initError; + goto _endJob; + } } } if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0); diff --git a/lib/zstd.h b/lib/zstd.h index 9ff8ad99..8ac56488 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -963,10 +963,6 @@ typedef enum { ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */ - /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */ - ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e. - * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */ - /* multi-threading parameters */ ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1) * More threads improve speed, but also increase memory usage. @@ -1011,19 +1007,19 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo * meaning "return to no-dictionary mode". * Note 1 : `dict` content will be copied internally. Use * ZSTD_CCtx_loadDictionary_byReference() to reference dictionary - * content instead. + * content instead. The dictionary buffer must then outlive its + * users. * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters. * For this reason, compression parameters cannot be changed anymore after loading a dictionary. * It's also a CPU-heavy operation, with non-negligible impact on latency. * Note 3 : Dictionary will be used for all future compression jobs. - * To return to "no-dictionary" situation, load a NULL dictionary */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_loadDictionary_byReference() : - * Same as ZSTD_CCtx_loadDictionary() except dictionary content will be - * referenced, instead of copied. The dictionary buffer must outlive its users. + * To return to "no-dictionary" situation, load a NULL dictionary + * Note 5 : Use ZSTD_CCtx_loadDictionary_advanced() to select how dictionary + * content will be interpreted. */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /*! ZSTD_CCtx_refCDict() : @@ -1050,8 +1046,11 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); * Note 1 : Prefix buffer is referenced. It must outlive compression job. * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters. * It's a CPU-heavy operation, with non-negligible impact on latency. - * Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */ + * Note 3 : By default, the prefix is treated as raw content + * (ZSTD_dm_rawContent). Use ZSTD_CCtx_refPrefix_advanced() to alter + * dictMode. */ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);