From e9853b2cdb69a2e6f462947030af94a28191aa0e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Aug 2015 19:07:32 +0100 Subject: [PATCH] Fixed : ZSTD_compress() can attempt compression on a too small buffer --- Makefile | 2 +- lib/fse.c | 22 +++++++++++----------- lib/zstd.c | 7 +++---- programs/Makefile | 4 ++-- programs/fuzzer.c | 18 ++++++++++++++---- 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 8049649d..6591e26d 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION=0.0.2 +export VERSION=0.1.0 export RELEASE=r$(VERSION) DESTDIR?= diff --git a/lib/fse.c b/lib/fse.c index 63684fc0..a82baa41 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -1132,7 +1132,7 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) size_t FSE_initCStream(FSE_CStream_t* bitC, void* start, size_t maxSize) { - if (maxSize < 8) return (size_t)-FSE_ERROR_dstSize_tooSmall; + if (maxSize < sizeof(bitC->ptr)) return (size_t)-FSE_ERROR_dstSize_tooSmall; bitC->bitContainer = 0; bitC->bitPos = 0; bitC->startPtr = (char*)start; @@ -1186,12 +1186,9 @@ void FSE_flushBits(FSE_CStream_t* bitC) size_t nbBytes = bitC->bitPos >> 3; FSE_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; - if (bitC->ptr <= bitC->endPtr) - { - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; - return; - } + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; } void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* statePtr) @@ -1208,7 +1205,7 @@ size_t FSE_closeCStream(FSE_CStream_t* bitC) FSE_addBitsFast(bitC, 1, 1); FSE_flushBits(bitC); - if (bitC->bitPos > 7) /* still some data to flush => too close to buffer's end */ + if (bitC->ptr >= bitC->endPtr) /* too close to buffer's end */ return 0; /* not compressible */ endPtr = bitC->ptr; @@ -1887,7 +1884,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 U16 nodeNb = STARTNODE; U32 nodeRoot; - // check + /* safety checks */ if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC; memset(huffNode0, 0, sizeof(huffNode0)); @@ -1976,7 +1973,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size /* init */ op += 6; /* jump Table -- could be optimized by delta / deviation */ - errorCode = FSE_initCStream(&bitC, op, dstSize); + errorCode = FSE_initCStream(&bitC, op, oend-op); if (FSE_isError(errorCode)) return 0; n = srcSize & ~15; // mod 16 @@ -2124,7 +2121,10 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize op += errorCode; /* Compress */ - op += HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); + errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); + if (FSE_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + op += errorCode; /* check compressibility */ if ((size_t)(op-ostart) >= srcSize-1) diff --git a/lib/zstd.c b/lib/zstd.c index 181eaf8f..35474b12 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -912,7 +912,7 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const } -size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) +size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) { /* Sanity check */ if (maxDstSize < ZSTD_frameHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; @@ -1081,7 +1081,6 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize) static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + maxDstSize; BYTE* op = ostart; /* Header */ @@ -1094,7 +1093,7 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co /* Compression */ { - size_t cSize = ZSTD_compressContinue(ctx, op, oend-op, src, srcSize); + size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); if (ZSTD_isError(cSize)) return cSize; op += cSize; maxDstSize -= cSize; @@ -1102,7 +1101,7 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co /* Close frame */ { - size_t endSize = ZSTD_compressEnd(ctx, op, oend-op); + size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); if(ZSTD_isError(endSize)) return endSize; op += endSize; } diff --git a/programs/Makefile b/programs/Makefile index 20205b90..1f7bdd5f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE?= v0.0.2 +RELEASE?= v0.1.0 DESTDIR?= PREFIX ?= /usr @@ -61,7 +61,7 @@ default: zstd all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen -zstd: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c +zstd : $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c $(CC) $(FLAGS) $^ -o $@$(EXT) zstd32: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c diff --git a/programs/fuzzer.c b/programs/fuzzer.c index db4bc65f..658a0cc6 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -323,7 +323,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; - (void)startTest; (void)compressibility; /* allocation */ srcBuffer = (BYTE*)malloc (srcBufferSize); @@ -332,7 +331,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled"); /* Create initial sample */ - FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed); + FUZ_generateSynthetic(srcBuffer, srcBufferSize, compressibility, &coreSeed); /* catch up testNb */ for (testNb=0; testNb < startTest; testNb++) @@ -356,10 +355,20 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0); - /* compression tests*/ + /* compression test */ cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize); CHECK(ZSTD_isError(cSize), "ZSTD_compress failed"); + /* compression failure test */ + { + size_t errorCode; + void* dBufferTooSmall = malloc(cSize-1); /* valgrind should catch overflows */ + if (dBufferTooSmall==NULL) { DISPLAY("not enough memory !"); exit(1); } + errorCode = ZSTD_compress(dBufferTooSmall, cSize-1, srcBuffer + sampleStart, sampleSize); + CHECK(!ZSTD_isError(errorCode), "ZSTD_compress should have failed ! (buffer too small)"); + free(dBufferTooSmall); + } + /* decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); @@ -393,8 +402,9 @@ int FUZ_usage(char* programName) DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); DISPLAY( " -h : display help and exit\n"); return 0; }