diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 0da95078..1cccca28 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -246,11 +246,14 @@ struct ZSTD_CCtx_params_s { /* Dictionary */ ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */ U32 dictContentByRef; - U32 nbThreads; /* Multithreading: used only to set mtctx parameters */ + U32 nbThreads; unsigned jobSize; unsigned overlapSizeLog; + + /* Test parameter */ + U32 testParam; }; typedef struct { diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4f10aedf..051a848a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -261,6 +261,7 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned #define ZSTD_CLEVEL_CUSTOM 999 +#if 0 static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) { if (cctx->requestedParams.compressionLevel==ZSTD_CLEVEL_CUSTOM) return; @@ -269,6 +270,7 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) cctx->pledgedSrcSizePlusOne-1, 0); cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; } +#endif ZSTD_CCtx_params* ZSTD_createCCtxParams(void) { @@ -280,6 +282,14 @@ ZSTD_CCtx_params* ZSTD_createCCtxParams(void) return params; } +size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params) +{ + if (!params) { return ERROR(GENERIC); } + memset(params, 0, sizeof(ZSTD_CCtx_params)); + params->compressionLevel = ZSTD_CLEVEL_DEFAULT; + return 0; +} + size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, ZSTD_compressionParameters cParams) { @@ -330,102 +340,27 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v switch(param) { - case ZSTD_p_compressionLevel : - if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel(); /* cap max compression level */ + case ZSTD_p_compressionLevel: + case ZSTD_p_windowLog: + case ZSTD_p_hashLog: + case ZSTD_p_chainLog: + case ZSTD_p_searchLog: + case ZSTD_p_minMatch: + case ZSTD_p_targetLength: + case ZSTD_p_compressionStrategy: if (value == 0) return 0; /* special value : 0 means "don't change anything" */ if (cctx->cdict) return ERROR(stage_wrong); - cctx->requestedParams.compressionLevel = value; - return 0; + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_windowLog : - DEBUGLOG(5, "setting ZSTD_p_windowLog = %u (cdict:%u)", - value, (cctx->cdict!=NULL)); - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.windowLog = value; - return 0; + case ZSTD_p_contentSizeFlag: + case ZSTD_p_checksumFlag: + case ZSTD_p_dictIDFlag: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_hashLog : - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.hashLog = value; - return 0; - - case ZSTD_p_chainLog : - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.chainLog = value; - return 0; - - case ZSTD_p_searchLog : - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.searchLog = value; - return 0; - - case ZSTD_p_minMatch : - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.searchLength = value; - return 0; - - case ZSTD_p_targetLength : - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.targetLength = value; - return 0; - - case ZSTD_p_compressionStrategy : - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); - ZSTD_cLevelToCParams(cctx); - cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value; - return 0; - - case ZSTD_p_contentSizeFlag : - DEBUGLOG(5, "set content size flag = %u", (value>0)); - /* Content size written in frame header _when known_ (default:1) */ - cctx->requestedParams.fParams.contentSizeFlag = value>0; - return 0; - - case ZSTD_p_checksumFlag : - /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ - cctx->requestedParams.fParams.checksumFlag = value>0; - return 0; - - case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ - DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); - cctx->requestedParams.fParams.noDictIDFlag = (value==0); - return 0; - - /* Dictionary parameters */ - case ZSTD_p_dictMode : + case ZSTD_p_dictMode: + case ZSTD_p_refDictContent: if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ - /* restrict dictionary mode, to "rawContent" or "fullDict" only */ - ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent); - if (value > (unsigned)ZSTD_dm_fullDict) - return ERROR(parameter_outOfBound); - cctx->requestedParams.dictMode = (ZSTD_dictMode_e)value; - return 0; - - case ZSTD_p_refDictContent : - if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ - /* dictionary content will be referenced, instead of copied */ - cctx->requestedParams.dictContentByRef = value>0; - return 0; + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, * even when referencing into Dictionary content @@ -462,6 +397,11 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v assert(cctx->mtctx != NULL); return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value); + case ZSTD_p_test : + DEBUGLOG(2, "Setting test parameter = %u", value); + cctx->requestedParams.testParam = (value > 0); + return 0; + default: return ERROR(parameter_unsupported); } } @@ -527,14 +467,18 @@ size_t ZSTD_CCtxParam_setParameter( return 0; case ZSTD_p_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(5, "set content size flag = %u", (value>0)); params->fParams.contentSizeFlag = value > 0; return 0; case ZSTD_p_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ params->fParams.checksumFlag = value > 0; return 0; case ZSTD_p_dictIDFlag : + DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); params->fParams.noDictIDFlag = (value == 0); return 0; @@ -559,7 +503,7 @@ size_t ZSTD_CCtxParam_setParameter( #ifndef ZSTD_MULTITHREAD if (value > 1) return ERROR(parameter_unsupported); #endif - // Do checks when applying parameters to cctx. + /* Do checks when applying params to cctx */ params->nbThreads = value; return 0; @@ -569,18 +513,57 @@ size_t ZSTD_CCtxParam_setParameter( return 0; case ZSTD_p_overlapSizeLog : + if (params->nbThreads <= 1) { return ERROR(parameter_unsupported); } params->overlapSizeLog = value; return 0; + case ZSTD_p_test : + DEBUGLOG(2, "setting opaque: ZSTD_p_test: %u", value); + params->testParam = (value > 0); + return 0; + default: return ERROR(parameter_unsupported); } } +static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params) +{ + DEBUGLOG(2, "======CCtxParams======"); + DEBUGLOG(2, "cParams: %u %u %u %u %u %u %u", + params->cParams.windowLog, + params->cParams.chainLog, + params->cParams.hashLog, + params->cParams.searchLog, + params->cParams.searchLength, + params->cParams.targetLength, + params->cParams.strategy); + DEBUGLOG(2, "fParams: %u %u %u", + params->fParams.contentSizeFlag, + params->fParams.checksumFlag, + params->fParams.noDictIDFlag); + DEBUGLOG(2, "cLevel, forceWindow: %u %u", + params->compressionLevel, + params->forceWindow); + DEBUGLOG(2, "dictionary: %u %u", + params->dictMode, + params->dictContentByRef); + DEBUGLOG(2, "multithreading: %u %u %u", + params->nbThreads, + params->jobSize, + params->overlapSizeLog); + DEBUGLOG(2, "testParam: %u", + params->testParam); +} + // This function should probably be updated whenever ZSTD_CCtx_params is updated. ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params) { + if (params == NULL) { return ERROR(GENERIC); } if (cctx->cdict) { return ERROR(stage_wrong); } + DEBUGLOG(2, "Applying cctx params\n"); + ZSTD_debugPrintCCtxParams(params); + /* Assume the compression and frame parameters are validated */ cctx->requestedParams.cParams = params->cParams; cctx->requestedParams.fParams = params->fParams; @@ -596,9 +579,14 @@ ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* /* Set multithreading parameters explicitly */ CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) ); - CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) ); - CHECK_F( ZSTD_CCtx_setParameter( + if (params->nbThreads > 1) { + CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) ); + CHECK_F( ZSTD_CCtx_setParameter( cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) ); + + } + /* Copy test parameter */ + cctx->requestedParams.testParam = params->testParam; return 0; } @@ -790,8 +778,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel) size_t ZSTD_estimateCStreamSize_advanced_opaque(ZSTD_CCtx_params* params) { if (params == NULL) { return 0; } - { - size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params); + { size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; @@ -3265,7 +3252,6 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ? cctx->appliedParams.cParams : ZSTD_getCParams(cLevel, 0, 0); - DEBUGLOG(2, "ZSTD_getBlockSize: cLevel %u\n", cLevel); return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog); } @@ -3485,7 +3471,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode); } -size_t ZSTD_compressBegin_advanced_opaque(ZSTD_CCtx* cctx, +size_t ZSTD_compressBegin_advanced_opaque( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) @@ -3619,11 +3606,12 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, } /* Internal */ -size_t ZSTD_compress_advanced_opaque(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_CCtx_params params) +size_t ZSTD_compress_advanced_opaque( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_CCtx_params params) { CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, srcSize, ZSTDb_not_buffered) ); @@ -3751,11 +3739,11 @@ static size_t ZSTD_initCDict_internal( } #endif -ZSTD_CDict* ZSTD_createCDict_advanced_opaque( +static ZSTD_CDict* ZSTD_createCDict_advanced_opaque( const void* dictBuffer, size_t dictSize, ZSTD_CCtx_params params, ZSTD_customMem customMem) { - DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); + DEBUGLOG(5, "ZSTD_createCDict_advanced_opaque, mode %u", (U32)params.dictMode); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); @@ -3794,31 +3782,6 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, cctxParams.dictMode = dictMode; cctxParams.dictContentByRef = byReference; return ZSTD_createCDict_advanced_opaque(dictBuffer, dictSize, cctxParams, customMem); -#if 0 - DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - - { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); - - if (!cdict || !cctx) { - ZSTD_free(cdict, customMem); - ZSTD_freeCCtx(cctx); - return NULL; - } - cdict->refContext = cctx; - - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, - dictBuffer, dictSize, - byReference, dictMode, - cParams) )) { - ZSTD_freeCDict(cdict); - return NULL; - } - - return cdict; - } -#endif } ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) @@ -4064,7 +4027,6 @@ size_t ZSTD_initCStream_internal_opaque(ZSTD_CStream* zcs, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { - DEBUGLOG(5, "ZSTD_initCStream_internal"); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ @@ -4145,9 +4107,9 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, { ZSTD_CCtx_params cctxParams = zcs->requestedParams; CHECK_F( ZSTD_checkCParams(params.cParams) ); - zcs->requestedParams.cParams = params.cParams; - zcs->requestedParams.fParams = params.fParams; - zcs->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; return ZSTD_initCStream_internal_opaque(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index ea5828f3..1296cd31 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -62,7 +62,7 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void) DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ elapsedTime, #mutex); \ } } \ - } else pthread_mutex_lock(mutex); \ + } else { pthread_mutex_lock(mutex); } \ } #else @@ -646,7 +646,7 @@ static size_t ZSTDMT_compress_advanced_opaque( } } /* for (chunkID=0; chunkID dstCapacity) { diff --git a/lib/zstd.h b/lib/zstd.h index 56316160..eb2a857f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -498,6 +498,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * TODO: ZSTD_estimateCCtxSize_advanced_opaque() * Note : CCtx estimation is only correct for single-threaded compression */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams); @@ -509,6 +510,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * TODO: ZSTD_estimateCStreamSize_advanced_opaque * Note : CStream estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -525,11 +527,10 @@ ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t sr /*! ZSTD_estimate?DictSize() : * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). * ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced(). + * TODO: ZSTD_estimateCDictSize_advanced_opaque(), can set by reference * Note : dictionary created "byReference" are smaller */ ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference); - -// By reference ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced_opaque(size_t dictSize, ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference); @@ -613,12 +614,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque( ZSTD_CCtx_params* params); ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params); ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createAndInitCCtxParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); -ZSTDLIB_API - /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. @@ -647,6 +647,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params); + /*! ZSTD_compress_usingCDict_advanced() : * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, @@ -1002,6 +1003,7 @@ typedef enum { /* advanced parameters - may not remain available after API update */ ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, * even when referencing into Dictionary content (default:0) */ + ZSTD_p_test, } ZSTD_cParameter; diff --git a/programs/Makefile b/programs/Makefile index 2460a091..a192716a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -47,7 +47,8 @@ DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +ZSTD_DEBUG_FLAGS = -g -DZSTD_DEBUG=2 +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) $(ZSTD_DEBUG_FLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) diff --git a/programs/fileio.c b/programs/fileio.c index 1dd8008e..b32dd83e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -213,6 +213,8 @@ void FIO_setOverlapLog(unsigned overlapLog){ DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); g_overlapLog = overlapLog; } +static U32 g_testParamFlag = 0; +void FIO_setTestParamFlag(unsigned testParamFlag) { g_testParamFlag = testParamFlag; } /*-************************************* @@ -411,6 +413,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); /* dictionary */ CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); + + /* Test */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_test, g_testParamFlag) ); } #elif defined(ZSTD_MULTITHREAD) { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index b1268c1f..88d4e152 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -430,6 +430,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--testParam")) { FIO_setTestParamFlag(1); continue; } #ifdef ZSTD_GZCOMPRESS if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } #endif diff --git a/tests/Makefile b/tests/Makefile index 3734f773..55de2f58 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -25,7 +25,7 @@ PRGDIR = ../programs PYTHON ?= python3 TESTARTEFACT := versionsTest namespaceTest -DEBUGLEVEL= 1 +DEBUGLEVEL= 2 DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) @@ -169,6 +169,12 @@ datagen : $(PRGDIR)/datagen.c datagencli.c roundTripCrash : $(ZSTD_FILES) roundTripCrash.c $(CC) $(FLAGS) $^ -o $@$(EXT) +OPAQUEFILES := $(ZSTD_FILES) $(ZDICT_FILES) roundTripCrashOpaque.c +roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_CPP) +roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_LD) +roundTripCrashOpaque : $(OPAQUEFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) + longmatch : $(ZSTD_FILES) longmatch.c $(CC) $(FLAGS) $^ -o $@$(EXT) diff --git a/tests/fullbench.c b/tests/fullbench.c index 45ef2b6a..0c41aa3d 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -136,8 +136,10 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con buffIn.src = src; buffIn.size = srcSize; buffIn.pos = 0; + ZSTD_compressStream(g_cstream, &buffOut, &buffIn); ZSTD_endStream(g_cstream, &buffOut); + return buffOut.pos; } @@ -383,11 +385,13 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) const BYTE* ip = dstBuff; const BYTE* iend; size_t frameHeaderSize, cBlockSize; + ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; ip += frameHeaderSize; /* Skip frame Header */ + cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ if (bp.blockType != bt_compressed) { DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); @@ -395,6 +399,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) } iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ ip += ZSTD_blockHeaderSize; /* skip block header */ + ZSTD_decompressBegin(g_zdc); ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ g_cSize = iend-ip; diff --git a/tests/roundTripCrashOpaque.c b/tests/roundTripCrashOpaque.c new file mode 100644 index 00000000..f9b6e7f8 --- /dev/null +++ b/tests/roundTripCrashOpaque.c @@ -0,0 +1,203 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* + This program takes a file in input, + performs a zstd round-trip test (compression - decompress) + compares the result with original + and generates a crash (double free) on corruption detection. +*/ + +/*=========================================== +* Dependencies +*==========================================*/ +#include /* size_t */ +#include /* malloc, free, exit */ +#include /* fprintf */ +#include /* stat */ +#include /* stat */ +#include "xxhash.h" + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +/*=========================================== +* Macros +*==========================================*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + fprintf(stderr, \ + "Error=> %s: %s", \ + #f, ZSTD_getErrorName(err)); \ + exit(1); \ +} } + + +/** roundTripTest() : +* Compresses `srcBuff` into `compressedBuff`, +* then decompresses `compressedBuff` into `resultBuff`. +* @return : result of decompression, which should be == `srcSize` +* or an error code if either compression or decompression fails. +* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` +* for compression to be guaranteed to work */ +static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcBuffSize) +{ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); + ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; + ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 }; + + ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, 1); + ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, 1); + + ZSTD_CCtx_applyCCtxParams(cctx, cctxParams); + + CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); + + ZSTD_freeCCtxParams(cctxParams); + ZSTD_freeCCtx(cctx); + + return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos); +} + + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* ip1 = (const char*)buff1; + const char* ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + fprintf(stderr, "Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + fprintf(stderr, "Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + fprintf(stderr, "Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName) +{ + size_t const fileSize = getFileSize(fileName); + void* buffer = malloc(fileSize); + if (!buffer) { + fprintf(stderr, "not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize); + free (buffer); +} + +int main(int argCount, const char** argv) { + if (argCount < 2) { + fprintf(stderr, "Error : no argument : need input file \n"); + exit(9); + } + fileCheck(argv[1]); + fprintf(stderr, "no pb detected\n"); + return 0; +} diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 3e551e33..0281a406 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1494,6 +1494,309 @@ _output_error: goto _cleanup; } +static int fuzzerTests_newAPI_opaque(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) +{ + U32 const maxSrcLog = bigTests ? 24 : 22; + static const U32 maxSampleLog = 19; + size_t const srcBufferSize = (size_t)1<= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } + else { DISPLAYUPDATE(2, "\r%6u ", testNb); } + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime32; + + /* states full reset (deliberately not synchronized) */ + /* some issues can only happen when reusing states */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { + DISPLAYLEVEL(5, "Creating new context \n"); + ZSTD_freeCCtx(zc); + zc = ZSTD_createCCtx(); + CHECK(zc==NULL, "ZSTD_createCCtx allocation error"); + resetAllowed=0; + } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { + ZSTD_freeDStream(zd); + zd = ZSTD_createDStream(); + CHECK(zd==NULL, "ZSTD_createDStream allocation error"); + ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ + } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/ + if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ + && oldTestLog /* at least one test happened */ && resetAllowed) { + maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); + if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; + { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; + CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, compressionLevel)); + } + } else { + U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevelCandidate = (FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / 3))) + + 1; + U32 const cLevel = MIN(cLevelCandidate, cLevelMax); + maxTestSize = FUZ_rLogLength(&lseed, testLog); + oldTestLog = testLog; + /* random dictionary selection */ + dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + if (!dictSize) dict=NULL; + } + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); + + /* mess with compression parameters */ + cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1; + cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); + cParams = ZSTD_adjustCParams(cParams, 0, 0); + + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_windowLog, cParams.windowLog) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_hashLog, cParams.hashLog) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_chainLog, cParams.chainLog) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_searchLog, cParams.searchLog) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_minMatch, cParams.searchLength) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_targetLength, cParams.targetLength) ); + + /* unconditionally set, to be sync with decoder */ + /* mess with frame parameters */ + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize); + + if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) ); + /* multi-threading parameters */ + { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; + U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbThreads, nbThreads) ); + if (nbThreads > 1) { + U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) ); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) ); + } + } + + if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1) ); + + if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, FUZ_rand(&lseed) & 1) ); + + + /* Apply parameters */ + + CHECK_Z (ZSTD_CCtx_applyCCtxParams(zc, cctxParams) ); + + if (FUZ_rand(&lseed) & 1) { + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); + if (dict && dictSize) { + /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ + size_t const setError = ZSTD_CCtx_applyCCtxParams(zc, cctxParams); + CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_applyCCtxParams should have failed"); + } } else { + CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); + } + } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; + for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) { + /* compress random chunks into randomly sized dst buffers */ + size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); + size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); + size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush; + ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; + outBuff.size = outBuff.pos + dstBuffSize; + + CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); + DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n", + (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos)); + + XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); + memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); + totalTestSize += inBuff.pos; + } + + /* final frame epilogue */ + { size_t remainingToFlush = (size_t)(-1); + while (remainingToFlush) { + ZSTD_inBuffer inBuff = { NULL, 0, 0 }; + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); + size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); + outBuff.size = outBuff.pos + adjustedDstSize; + DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize); + remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); + CHECK(ZSTD_isError(remainingToFlush), + "ZSTD_compress_generic w/ ZSTD_e_end error : %s", + ZSTD_getErrorName(remainingToFlush) ); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = outBuff.pos; + DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize); + } + + /* multi - fragments decompression test */ + if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { + DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict); + CHECK_Z( ZSTD_resetDStream(zd) ); + } else { + DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize); + CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); + } + { size_t decompressionResult = 1; + ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + for (totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + inBuff.size = inBuff.pos + readCSrcSize; + outBuff.size = inBuff.pos + dstBuffSize; + DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n", + (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize); + decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); + CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize); + } + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); + CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn