From 00fd7a21101e50a928c5164a758c35b25994acf5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 28 Nov 2015 16:03:22 +0100 Subject: [PATCH] protect 32-bits mode --- lib/bitstream.h | 4 +-- lib/error.h | 6 ++-- lib/zstd_buffered.h | 1 + lib/zstd_compress.c | 3 +- lib/zstd_decompress.c | 5 ++- programs/Makefile | 71 ++++++++++++++++++++++--------------------- programs/paramgrill.c | 11 ++++--- 7 files changed, 55 insertions(+), 46 deletions(-) diff --git a/lib/bitstream.h b/lib/bitstream.h index e6c2228e..dcfe8b0a 100644 --- a/lib/bitstream.h +++ b/lib/bitstream.h @@ -208,7 +208,7 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ } MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) @@ -218,7 +218,7 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) bitC->ptr += nbBytes; if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ } /*! BIT_closeCStream diff --git a/lib/error.h b/lib/error.h index 86f3c384..cfdaf500 100644 --- a/lib/error.h +++ b/lib/error.h @@ -68,10 +68,10 @@ extern "C" { #define ERROR_LIST(ITEM) \ ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \ - ITEM(PREFIX(mode_unsupported)) ITEM(PREFIX(init_missing))\ - ITEM(PREFIX(memory_allocation)) \ + ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(frameParameter_unsupportedBy32bitsImplementation)) \ + ITEM(PREFIX(init_missing)) ITEM(PREFIX(memory_allocation)) \ ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \ - ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \ + ITEM(PREFIX(corruption_detected)) \ ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \ ITEM(PREFIX(maxCode)) diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index ecbc1df5..80ba819d 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -83,6 +83,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); * * ZBUFF_compressEnd() instructs to finish a frame. * It will perform a flush and write frame epilogue. +* Note that the epilogue is necessary for decoders to consider a frame completed. * Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small. * In which case, call again ZBUFF_compressFlush() to complete the flush. * @return : nb of bytes still present into internal buffer (0 if it's empty) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 5a825b84..67c97e53 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -143,6 +143,7 @@ void ZSTD_validateParams(ZSTD_parameters* params) const U32 btPlus = (params->strategy == ZSTD_btlazy2); /* validate params */ + if (MEM_32bits()) if (params->windowLog > 25) params->windowLog = 25; /* 32 bits mode cannot flush > 24 bits */ if (params->windowLog > ZSTD_WINDOWLOG_MAX) params->windowLog = ZSTD_WINDOWLOG_MAX; if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN; @@ -153,7 +154,7 @@ void ZSTD_validateParams(ZSTD_parameters* params) if (params->windowLog > srcLog) params->windowLog = srcLog; } - if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */ if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN; if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 122ff2b0..bd6cb099 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -204,8 +204,11 @@ size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcS * @return : 0, or an error code, which can be tested using ZSTD_isError() */ static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize) { + size_t result; if (srcSize != zc->headerSize) return ERROR(srcSize_wrong); - return ZSTD_getFrameParams(&(zc->params), src, srcSize); + result = ZSTD_getFrameParams(&(zc->params), src, srcSize); + if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bitsImplementation); + return result; } diff --git a/programs/Makefile b/programs/Makefile index de66b988..9d73cb95 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -43,8 +43,8 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -ZSTD_FILES = $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c -ZSTD_LEGACY = $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c +ZSTD_FILES := $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c +ZSTD_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c ifeq ($(ZSTD_LEGACY),disable) CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 @@ -151,83 +151,84 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest -test-zstd: zstd datagen +zstd-playTests: datagen @echo "\n**** frame concatenation **** " @echo "hello " > hello.tmp @echo "world!" > world.tmp @cat hello.tmp world.tmp > helloworld.tmp - ./zstd hello.tmp > hello.zstd - ./zstd world.tmp > world.zstd + $(ZSTD) hello.tmp > hello.zstd + $(ZSTD) world.tmp > world.zstd @cat hello.zstd world.zstd > helloworld.zstd - ./zstd -d helloworld.zstd > result.tmp + $(ZSTD) -df helloworld.zstd > result.tmp cat result.tmp sdiff helloworld.tmp result.tmp @rm *.tmp *.zstd @echo frame concatenation test completed @echo "**** flush write error test **** " - echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi + echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi + echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi @echo "**** zstd round-trip tests **** " @./datagen | md5sum > tmp1 - ./datagen | ./zstd -v | ./zstd -d | md5sum > tmp2 + ./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen | ./zstd -6 -v | ./zstd -d | md5sum > tmp2 + ./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g270000000 | md5sum > tmp1 - ./datagen -g270000000 | ./zstd -v | ./zstd -d | md5sum > tmp2 + ./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g270000000 | ./zstd -v2 | ./zstd -d | md5sum > tmp2 + ./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g270000000 | ./zstd -v3 | ./zstd -d | md5sum > tmp2 + ./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g140000000 -P60| md5sum > tmp1 - ./datagen -g140000000 -P60 | ./zstd -v4 | ./zstd -d | md5sum > tmp2 + ./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | ./zstd -v5 | ./zstd -d | md5sum > tmp2 + ./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | ./zstd -v6 | ./zstd -d | md5sum > tmp2 + ./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g70000000 -P70 | md5sum > tmp1 - ./datagen -g70000000 -P70 | ./zstd -v7 | ./zstd -d | md5sum > tmp2 + ./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | ./zstd -v8 | ./zstd -d | md5sum > tmp2 + ./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | ./zstd -v9 | ./zstd -d | md5sum > tmp2 + ./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g35000000 -P75 | md5sum > tmp1 - ./datagen -g35000000 -P75 | ./zstd -v10 | ./zstd -d | md5sum > tmp2 + ./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | ./zstd -v11 | ./zstd -d | md5sum > tmp2 + ./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | ./zstd -v12 | ./zstd -d | md5sum > tmp2 + ./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g18000000 -P80 | md5sum > tmp1 - ./datagen -g18000000 -P80 | ./zstd -v13 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | ./zstd -v14 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | ./zstd -v15 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | ./zstd -v16 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | ./zstd -v17 | ./zstd -d | md5sum > tmp2 + ./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g50000000 -P94 | md5sum > tmp1 - ./datagen -g50000000 -P94 | ./zstd -v18 | ./zstd -d | md5sum > tmp2 + ./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 - ./datagen -g50000000 -P94 | ./zstd -v19 | ./zstd -d | md5sum > tmp2 + ./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 @./datagen -g99000000 -P99 | md5sum > tmp1 - ./datagen -g99000000 -P99 | ./zstd -v20 | ./zstd -d | md5sum > tmp2 + ./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 ./datagen -g6000000000 -P99| md5sum > tmp1 - ./datagen -g6000000000 -P99| ./zstd -vq | ./zstd -d | md5sum > tmp2 + ./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2 @diff tmp1 tmp2 -test-zstd32: zstd32 datagen - ./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID) - ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) - ./datagen -g6GB -P99 | ./zstd32 -vq | ./zstd32 -d > $(VOID) +test-zstd: ZSTD = ./zstd +test-zstd: zstd zstd-playTests + +test-zstd32: ZSTD = ./zstd32 +test-zstd32: zstd32 zstd-playTests test-fullbench: fullbench datagen ./fullbench -i1 diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 62b2dcc2..0965a5c6 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -1161,12 +1161,15 @@ int main(int argc, char** argv) if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } - if (optimizer) - result = optimizeForSize(input_filename); - if (filenamesStart==0) result = benchSample(); - else result = benchFiles(argv+filenamesStart, argc-filenamesStart); + else + { + if (optimizer) + result = optimizeForSize(input_filename); + else + result = benchFiles(argv+filenamesStart, argc-filenamesStart); + } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }