diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 858eaca3..8d493852 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -241,6 +241,7 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) { + DEBUGLOG(4, "ZSTD_CCtx_setParameter"); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); switch(param) @@ -269,13 +270,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, - * even when referencing into Dictionary content + * even when referencing into Dictionary content. * default : 0 when using a CDict, 1 when using a Prefix */ - cctx->loadedDictEnd = 0; return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_nbThreads: - if (value > 1 && cctx->staticSize) { + if ((value > 1) && cctx->staticSize) { return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ } return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); @@ -320,6 +320,7 @@ size_t ZSTD_CCtxParam_setParameter( return CCtxParams->compressionLevel; case ZSTD_p_windowLog : + DEBUGLOG(4, "setting windowLog=%u", value); if (value) { /* 0 : does not change current windowLog */ CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); ZSTD_cLevelToCCtxParams(CCtxParams); @@ -459,13 +460,9 @@ size_t ZSTD_CCtxParam_setParameter( } } -/** - * This function should be updated whenever ZSTD_CCtx_params is updated. - * Parameters are copied manually before the dictionary is loaded. - * The multithreading parameters jobSize and overlapSizeLog are set only if - * nbThreads > 1. - * - * Pledged srcSize is treated as unknown. +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. */ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) @@ -473,27 +470,8 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong); - /* Assume the compression and frame parameters are validated */ - cctx->requestedParams.cParams = params->cParams; - cctx->requestedParams.fParams = params->fParams; - cctx->requestedParams.compressionLevel = params->compressionLevel; + cctx->requestedParams = *params; - /* Set force window explicitly since it sets cctx->loadedDictEnd */ - CHECK_F( ZSTD_CCtx_setParameter( - cctx, ZSTD_p_forceMaxWindow, params->forceWindow) ); - - /* Set multithreading parameters explicitly */ - CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) ); - if (params->nbThreads > 1) { - CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) ); - CHECK_F( ZSTD_CCtx_setParameter( - cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) ); - } - - /* Copy long distance matching parameters */ - cctx->requestedParams.ldmParams = params->ldmParams; - - /* customMem is used only for create/free params and can be ignored */ return 0; } @@ -2091,7 +2069,6 @@ size_t ZSTD_compressBegin_advanced_internal( unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal"); - DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag); /* compression parameters verification and optimization */ CHECK_F( ZSTD_checkCParams(params.cParams) ); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 2926d3b4..6b6466fe 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -140,7 +140,7 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) return poolSize + totalBufferSize; } -static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize) +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) { ZSTD_pthread_mutex_lock(&bufPool->poolMutex); DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); @@ -359,7 +359,7 @@ void ZSTDMT_compressChunk(void* jobDescription) assert(job->firstChunk); /* should only happen for first segment */ if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } } else { /* srcStart points at reloaded section */ - ZSTD_CCtx_params jobParams = job->params; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN; /* load dictionary in "content-only" mode (no header analysis) */ diff --git a/programs/fileio.c b/programs/fileio.c index c9b6b04e..101c99b1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -639,7 +639,6 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, memset(&prefs, 0, sizeof(prefs)); assert(blockSize <= ress->srcBufferSize); - assert(LZ4F_compressBound(blockSize) <= ress->dstBufferSize); prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; @@ -649,6 +648,7 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, #if LZ4_VERSION_NUMBER >= 10600 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; #endif + assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); { size_t readSize; diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 207b2e01..920e988b 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1482,43 +1482,47 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double cParams = ZSTD_adjustCParams(cParams, 0, 0); if (FUZ_rand(&lseed) & 1) { + DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) ); assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; - DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog); } if (FUZ_rand(&lseed) & 1) { - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) ); DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) ); } if (FUZ_rand(&lseed) & 1) { - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) ); DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) ); } if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) ); /* mess with long distance matching parameters */ - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) ); + if (bigTests) { + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) ); + } /* mess with frame parameters */ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); - DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize); + if (FUZ_rand(&lseed) & 1) { + DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + } /* multi-threading parameters */ { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) ); DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) ); if (nbThreads > 1) { U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) ); @@ -1526,10 +1530,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double } } - if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); /* Apply parameters */ if (useOpaqueAPI) { + DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb); CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) ); } @@ -1569,8 +1574,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double outBuff.size = outBuff.pos + dstBuffSize; CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); - DISPLAYLEVEL(6, "compress consumed %u bytes (total : %u) \n", - (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos)); + DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) \n", + testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos)); XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);