From 20d5e03893004cf236297808a0ee51a2a94d506b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Apr 2017 18:34:02 -0700 Subject: [PATCH] content size is controlled at bufferless level so it's active for all entry points Also : added relevant test (wrong content size) in fuzzer --- lib/common/error_private.c | 2 +- lib/compress/zstd_compress.c | 13 ++++++++++--- lib/compress/zstdmt_compress.c | 11 ++++++----- tests/fuzzer.c | 30 ++++++++++++++++++++++-------- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/lib/common/error_private.c b/lib/common/error_private.c index 44ae2010..b3287245 100644 --- a/lib/common/error_private.c +++ b/lib/common/error_private.c @@ -29,7 +29,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(memory_allocation): return "Allocation error : not enough memory"; case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(corruption_detected): return "Corrupted block detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0ce2054f..110698c2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -81,6 +81,7 @@ struct ZSTD_CCtx_s { size_t workSpaceSize; size_t blockSize; U64 frameContentSize; + U64 consumedSrcSize; XXH64_state_t xxhState; ZSTD_customMem customMem; @@ -241,6 +242,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra U32 const end = (U32)(cctx->nextSrc - cctx->base); cctx->params = params; cctx->frameContentSize = frameContentSize; + cctx->consumedSrcSize = 0; cctx->lowLimit = end; cctx->dictLimit = end; cctx->nextToUpdate = end+1; @@ -313,6 +315,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->params = params; zc->blockSize = blockSize; zc->frameContentSize = frameContentSize; + zc->consumedSrcSize = 0; { int i; for (i=0; irep[i] = repStartValue[i]; } if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { @@ -2493,6 +2496,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(cSize)) return cSize; + cctx->consumedSrcSize += srcSize; return cSize + fhSize; } else return fhSize; @@ -2503,7 +2507,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } @@ -2516,7 +2520,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const { size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } /*! ZSTD_loadDictionaryContent() : @@ -2767,10 +2771,13 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, const void* src, size_t srcSize) { size_t endResult; - size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); + size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */); if (ZSTD_isError(cSize)) return cSize; endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); if (ZSTD_isError(endResult)) return endResult; + if (cctx->params.fParams.contentSizeFlag) { /* control src size */ + if (cctx->frameContentSize != cctx->consumedSrcSize) return ERROR(srcSize_wrong); + } return cSize + endResult; } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index c9c3e853..50c98ae9 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -235,11 +235,12 @@ void ZSTDMT_compressChunk(void* jobDescription) if (job->cdict) DEBUGLOG(3, "using CDict "); if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } } else { /* srcStart points at reloaded section */ - size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1); /* Force loading dictionary in "content-only" mode (no header analysis) */ - size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize); - if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; } - ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1); - } + if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ + { size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1); /* Force loading dictionary in "content-only" mode (no header analysis) */ + size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize); + if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; } + ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1); + } } if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0); if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 342c953e..cdbbe8a2 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -524,9 +524,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ if (!ZSTD_isError(r)) goto _output_error; - if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; } + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); @@ -535,6 +535,22 @@ static int basicUnitTests(U32 seed, double compressibility) if (!ZSTD_isError(r)) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); + /* content size verification test */ + DISPLAYLEVEL(4, "test%3i : Content size verification : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const srcSize = 5000; + size_t const wrongSrcSize = (srcSize + 1000); + ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); + params.fParams.contentSizeFlag = 1; + { size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize); + if (ZSTD_isError(result)) goto _output_error; + } + { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); + if (!ZSTD_isError(result)) goto _output_error; + if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; + DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result)); + } } + /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); static const size_t dictSize = 65 KB; @@ -788,12 +804,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression tests */ - { - unsigned const cLevel = - (FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter))) + - 1; + { unsigned const cLevel = + ( FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) ) + + 1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));