From 1f0b09dc4c649c03e57b1904259da62e49a49ad6 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 14 Nov 2016 10:29:25 -0800 Subject: [PATCH 1/8] Fix travis-ci timeout. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9396c98d..8462b971 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ matrix: language: cpp install: - export CXX="g++-4.8" CC="gcc-4.8" + - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' addons: apt: sources: From 324c8ab000ee7b8fc48722c45f6a1bd58eb2f2ca Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 14 Nov 2016 11:02:03 -0800 Subject: [PATCH 2/8] [pzstd] Remove gtest dependency from make all --- .travis.yml | 2 +- appveyor.yml | 3 ++- contrib/pzstd/Makefile | 18 +++++++++++------- contrib/pzstd/README.md | 5 +++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8462b971..73ff0486 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: packages: - gcc-4.8 - g++-4.8 - env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean" + env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" diff --git a/appveyor.yml b/appveyor.yml index fbdc30c4..bce87a3a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,7 +51,8 @@ build_script: ECHO *** Building pzstd for %PLATFORM% && ECHO *** && make -C contrib\pzstd googletest-mingw64 && - make -C contrib\pzstd all && + make -C contrib\pzstd pzstd.exe && + make -C contrib\pzstd tests && make -C contrib\pzstd check && make -C contrib\pzstd clean ) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index ad0e5567..99d955e9 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -34,7 +34,7 @@ LDFLAGS ?= PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. GTEST_INC = -isystem googletest/googletest/include -PZSTD_CPPFLAGS = $(PZSTD_INC) $(GTEST_INC) +PZSTD_CPPFLAGS = $(PZSTD_INC) PZSTD_CCXXFLAGS = PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS) PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) @@ -47,10 +47,10 @@ ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS) # gtest libraries need to go before "-lpthread" because they depend on it. GTEST_LIB = -L googletest/build/googlemock/gtest -LIBS = $(GTEST_LIB) -lpthread +LIBS = # Compilation commands -LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -o $@ +LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -lpthread -o $@ CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@ CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@ @@ -109,7 +109,7 @@ uninstall: # Targets for many different builds .PHONY: all all: PZSTD_CPPFLAGS += -DNDEBUG -all: pzstd$(EXT) tests roundtrip +all: pzstd$(EXT) .PHONY: debug debug: EXTRA_FLAGS += -g @@ -130,7 +130,7 @@ ubsan: debug .PHONY: all32 all32: EXTRA_FLAGS += -m32 -all32: all +all32: all tests roundtrip .PHONY: debug32 debug32: EXTRA_FLAGS += -m32 @@ -177,12 +177,14 @@ test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \ Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a $(LD_COMMAND) -test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main +test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB) +test/%Test$(EXT): LIBS += -lgtest -lgtest_main test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \ SkippableFrame.o $(ZSTDDIR)/libzstd.a $(LD_COMMAND) -utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main +utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB) +utils/test/%Test$(EXT): LIBS += -lgtest -lgtest_main utils/test/%Test$(EXT): utils/test/%Test.o $(LD_COMMAND) @@ -233,10 +235,12 @@ $(PROGDIR)/%.o: $(PROGDIR)/%.c $(CXX_COMMAND) $(POSTCOMPILE) +test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC) test/%.o: test/%.cpp $(CXX_COMMAND) $(POSTCOMPILE) +utils/test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC) utils/test/%.o: utils/test/%.cpp $(CXX_COMMAND) $(POSTCOMPILE) diff --git a/contrib/pzstd/README.md b/contrib/pzstd/README.md index 3fe7b0b9..84d94581 100644 --- a/contrib/pzstd/README.md +++ b/contrib/pzstd/README.md @@ -51,5 +51,6 @@ Pigz cannot do parallel decompression, it simply does each of reading, decompres ## Tests Tests require that you have [gtest](https://github.com/google/googletest) installed. -Modify `GTEST_INC` and `GTEST_LIB` in `test/Makefile` and `utils/test/Makefile` to work for your install of gtest. -Then run `make test` in the `contrib/pzstd` directory. +Set `GTEST_INC` and `GTEST_LIB` in `Makefile` to specify the location of the gtest headers and libraries. +Alternatively, run `make googletest`, which will clone googletest and build it. +Run `make tests && make check` to run tests. From 24701de877a486b2f59a869e0d38d5a697a5d7d8 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 14 Nov 2016 11:33:37 -0800 Subject: [PATCH 3/8] Fix uninitialized memory read --- lib/decompress/zstd_decompress.c | 21 +++++++++------------ lib/legacy/zstd_v02.c | 13 +++++-------- lib/legacy/zstd_v03.c | 13 +++++-------- lib/legacy/zstd_v04.c | 15 ++++++--------- lib/legacy/zstd_v05.c | 17 +++++++---------- lib/legacy/zstd_v06.c | 17 +++++++---------- lib/legacy/zstd_v07.c | 17 +++++++---------- 7 files changed, 46 insertions(+), 67 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b6a3865c..e8d6b152 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -104,7 +104,6 @@ struct ZSTD_DCtx_s U32 dictID; const BYTE* litPtr; ZSTD_customMem customMem; - size_t litBufSize; size_t litSize; size_t rleSize; BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; @@ -429,10 +428,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } @@ -459,13 +458,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litSize+lhSize > srcSize) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; - dctx->litBufSize = srcSize-lhSize; dctx->litSize = litSize; return lhSize+litSize; } @@ -492,8 +490,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: @@ -867,7 +865,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) FORCE_NOINLINE size_t ZSTD_execSequenceLast7(BYTE* op, BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE** litPtr, const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; @@ -879,7 +877,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op, /* check */ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ /* copy literals */ @@ -914,7 +912,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op, FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE** litPtr, const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; @@ -926,8 +924,8 @@ size_t ZSTD_execSequence(BYTE* op, /* check */ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd); + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); /* copy Literals */ ZSTD_copy8(op, *litPtr); @@ -1002,7 +1000,6 @@ static size_t ZSTD_decompressSequences( BYTE* const oend = ostart + maxDstSize; BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); @@ -1028,7 +1025,7 @@ static size_t ZSTD_decompressSequences( for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; { seq_t const sequence = ZSTD_decodeSequence(&seqState); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } } diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index 24498fed..acaa4632 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s blockType_t bType; U32 phase; const BYTE* litPtr; - size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */ @@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, size_t litSize = BLOCKSIZE; const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return readSize; /* works if it's an error too */ } case IS_RAW: @@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, if (litSize > srcSize-3) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return litSize+3; } /* direct reference into compressed stream */ dctx->litPtr = istart+3; - dctx->litBufSize = srcSize-3; dctx->litSize = litSize; return litSize+3; } @@ -2968,8 +2966,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, if (litSize > BLOCKSIZE) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[3], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return 4; } } @@ -3175,7 +3173,7 @@ static size_t ZSTD_execSequence(BYTE* op, /* checks */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ @@ -3241,7 +3239,6 @@ static size_t ZSTD_decompressSequences( BYTE* const oend = ostart + maxDstSize; size_t errorCode, dumpsLength; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litMax = litPtr + dctx->litBufSize; const BYTE* const litEnd = litPtr + dctx->litSize; int nbSeq; const BYTE* dumps; @@ -3277,7 +3274,7 @@ static size_t ZSTD_decompressSequences( size_t oneSeqSize; nbSeq--; ZSTD_decodeSequence(&sequence, &seqState); - oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend); + oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index a3bd1da2..48803ea0 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s blockType_t bType; U32 phase; const BYTE* litPtr; - size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */ @@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, size_t litSize = BLOCKSIZE; const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return readSize; /* works if it's an error too */ } case IS_RAW: @@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, if (litSize > srcSize-3) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return litSize+3; } /* direct reference into compressed stream */ dctx->litPtr = istart+3; - dctx->litBufSize = srcSize-3; dctx->litSize = litSize; return litSize+3; } @@ -2609,8 +2607,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, if (litSize > BLOCKSIZE) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[3], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return 4; } } @@ -2816,7 +2814,7 @@ static size_t ZSTD_execSequence(BYTE* op, /* checks */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ @@ -2882,7 +2880,6 @@ static size_t ZSTD_decompressSequences( BYTE* const oend = ostart + maxDstSize; size_t errorCode, dumpsLength; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litMax = litPtr + dctx->litBufSize; const BYTE* const litEnd = litPtr + dctx->litSize; int nbSeq; const BYTE* dumps; @@ -2918,7 +2915,7 @@ static size_t ZSTD_decompressSequences( size_t oneSeqSize; nbSeq--; ZSTD_decodeSequence(&sequence, &seqState); - oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend); + oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 0a740bac..c0738b30 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s blockType_t bType; ZSTD_dStage stage; const BYTE* litPtr; - size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */]; BYTE headerBuffer[ZSTD_frameHeaderSize_max]; @@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t litSize = BLOCKSIZE; const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return readSize; /* works if it's an error too */ } case IS_RAW: @@ -2859,13 +2858,12 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litSize > srcSize-3) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return litSize+3; } /* direct reference into compressed stream */ dctx->litPtr = istart+3; - dctx->litBufSize = srcSize-3; dctx->litSize = litSize; return litSize+3; } case IS_RLE: @@ -2874,8 +2872,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litSize > BLOCKSIZE) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[3], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); return 4; } default: @@ -3069,7 +3067,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) static size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_8, + const BYTE** litPtr, const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) { static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ @@ -3084,7 +3082,7 @@ static size_t ZSTD_execSequence(BYTE* op, /* check */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */ /* copy Literals */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ @@ -3167,7 +3165,6 @@ static size_t ZSTD_decompressSequences( BYTE* const oend = ostart + maxDstSize; size_t errorCode, dumpsLength; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8; const BYTE* const litEnd = litPtr + dctx->litSize; int nbSeq; const BYTE* dumps; @@ -3206,7 +3203,7 @@ static size_t ZSTD_decompressSequences( size_t oneSeqSize; nbSeq--; ZSTD_decodeSequence(&sequence, &seqState); - oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); + oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index f63a97fd..02ba8906 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2731,7 +2731,6 @@ struct ZSTDv05_DCtx_s ZSTDv05_dStage stage; U32 flagStaticTables; const BYTE* litPtr; - size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTDv05_frameHeaderSize_max]; @@ -2978,8 +2977,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case IS_PCH: @@ -3002,8 +3001,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, if (HUFv05_isError(errorCode)) return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case IS_RAW: @@ -3028,13 +3027,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, if (litSize+lhSize > srcSize) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; - dctx->litBufSize = srcSize-lhSize; dctx->litSize = litSize; return lhSize+litSize; } @@ -3059,8 +3057,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, if (litSize > BLOCKSIZE) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: @@ -3289,7 +3287,7 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState) static size_t ZSTDv05_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_8, + const BYTE** litPtr, const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) { static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ @@ -3304,7 +3302,7 @@ static size_t ZSTDv05_execSequence(BYTE* op, /* check */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */ /* copy Literals */ ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ @@ -3378,7 +3376,6 @@ static size_t ZSTDv05_decompressSequences( BYTE* const oend = ostart + maxDstSize; size_t errorCode, dumpsLength; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8; const BYTE* const litEnd = litPtr + dctx->litSize; int nbSeq; const BYTE* dumps; @@ -3416,7 +3413,7 @@ static size_t ZSTDv05_decompressSequences( size_t oneSeqSize; nbSeq--; ZSTDv05_decodeSequence(&sequence, &seqState); - oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); + oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 88be4943..01546f4e 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -2893,7 +2893,6 @@ struct ZSTDv06_DCtx_s ZSTDv06_dStage stage; U32 flagRepeatTable; const BYTE* litPtr; - size_t litBufSize; size_t litSize; BYTE litBuffer[ZSTDv06_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX]; @@ -3170,8 +3169,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case IS_PCH: @@ -3192,8 +3191,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, if (HUFv06_isError(errorCode)) return ERROR(corruption_detected); } dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case IS_RAW: @@ -3217,13 +3216,12 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, if (litSize+lhSize > srcSize) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; - dctx->litBufSize = srcSize-lhSize; dctx->litSize = litSize; return lhSize+litSize; } @@ -3247,8 +3245,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: @@ -3438,7 +3436,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState) size_t ZSTDv06_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_8, + const BYTE** litPtr, const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; @@ -3451,7 +3449,7 @@ size_t ZSTDv06_execSequence(BYTE* op, /* check */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ /* copy Literals */ ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ @@ -3523,7 +3521,6 @@ static size_t ZSTDv06_decompressSequences( BYTE* const oend = ostart + maxDstSize; BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8; const BYTE* const litEnd = litPtr + dctx->litSize; FSEv06_DTable* DTableLL = dctx->LLTable; FSEv06_DTable* DTableML = dctx->MLTable; @@ -3567,7 +3564,7 @@ static size_t ZSTDv06_decompressSequences( pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset); #endif - { size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); + { size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); if (ZSTDv06_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index c7693f24..1050080b 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3021,7 +3021,6 @@ struct ZSTDv07_DCtx_s U32 dictID; const BYTE* litPtr; ZSTDv07_customMem customMem; - size_t litBufSize; size_t litSize; BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX]; @@ -3395,9 +3394,9 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8; dctx->litSize = litSize; dctx->litEntropy = 1; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case lbt_repeat: @@ -3418,8 +3417,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, if (HUFv07_isError(errorCode)) return ERROR(corruption_detected); } dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case lbt_raw: @@ -3443,13 +3442,12 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, if (litSize+lhSize > srcSize) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; - dctx->litBufSize = srcSize-lhSize; dctx->litSize = litSize; return lhSize+litSize; } @@ -3473,8 +3471,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: @@ -3662,7 +3660,7 @@ static seq_t ZSTDv07_decodeSequence(seqState_t* seqState) static size_t ZSTDv07_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE** litPtr, const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; @@ -3674,7 +3672,7 @@ size_t ZSTDv07_execSequence(BYTE* op, /* check */ if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ /* copy Literals */ ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ @@ -3746,7 +3744,6 @@ static size_t ZSTDv07_decompressSequences( BYTE* const oend = ostart + maxDstSize; BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; const BYTE* const litEnd = litPtr + dctx->litSize; FSEv07_DTable* DTableLL = dctx->LLTable; FSEv07_DTable* DTableML = dctx->MLTable; @@ -3776,7 +3773,7 @@ static size_t ZSTDv07_decompressSequences( for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) { nbSeq--; { seq_t const sequence = ZSTDv07_decodeSequence(&seqState); - size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); + size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } } From 4359d21ad7281f77bfba8003d3da40d9157ef05f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 14 Nov 2016 17:52:51 -0800 Subject: [PATCH 4/8] Merge two memset() calls into one --- lib/decompress/zstd_decompress.c | 3 +-- lib/legacy/zstd_v02.c | 3 +-- lib/legacy/zstd_v03.c | 3 +-- lib/legacy/zstd_v04.c | 3 +-- lib/legacy/zstd_v05.c | 3 +-- lib/legacy/zstd_v06.c | 3 +-- lib/legacy/zstd_v07.c | 3 +-- 7 files changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e8d6b152..cc92ca27 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -488,10 +488,9 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; } if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index acaa4632..ed082aad 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -2964,10 +2964,9 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, { const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ if (litSize > BLOCKSIZE) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[3], litSize); + memset(dctx->litBuffer, istart[3], litSize + 8); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, 8); return 4; } } diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 48803ea0..32145067 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -2605,10 +2605,9 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, { const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ if (litSize > BLOCKSIZE) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[3], litSize); + memset(dctx->litBuffer, istart[3], litSize + 8); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, 8); return 4; } } diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index c0738b30..11b5481a 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -2870,10 +2870,9 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, { const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ if (litSize > BLOCKSIZE) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[3], litSize); + memset(dctx->litBuffer, istart[3], litSize + 8); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, 8); return 4; } default: diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 02ba8906..bf1235a3 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3055,10 +3055,9 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, break; } if (litSize > BLOCKSIZE) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 01546f4e..6584d485 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3243,10 +3243,9 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, break; } if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 1050080b..2ae6c5ad 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3469,10 +3469,9 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, break; } if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+1; } default: From f147fccd0c5da08ab05995b85922444f575c9183 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 15 Nov 2016 16:39:09 -0800 Subject: [PATCH 5/8] [pzstd] Fix frame size for small files + add logging --- contrib/pzstd/Pzstd.cpp | 15 ++++++--------- contrib/pzstd/Pzstd.h | 12 ++++++++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index c5b4ce4c..1778fef2 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -341,15 +341,7 @@ static size_t calculateStep( std::uintmax_t size, size_t numThreads, const ZSTD_parameters ¶ms) { - size_t step = size_t{1} << (params.cParams.windowLog + 2); - // If file size is known, see if a smaller step will spread work more evenly - if (size != 0) { - const std::uintmax_t newStep = size / numThreads; - if (newStep != 0 && newStep <= std::numeric_limits::max()) { - step = std::min(step, static_cast(newStep)); - } - } - return step; + return size_t{1} << (params.cParams.windowLog + 2); } namespace { @@ -401,6 +393,7 @@ std::uint64_t asyncCompressChunks( // Break the input up into chunks of size `step` and compress each chunk // independently. size_t step = calculateStep(size, numThreads, params); + state.log(DEBUG, "Chosen frame size: %zu\n", step); auto status = FileStatus::Continue; while (status == FileStatus::Continue && !state.errorHolder.hasError()) { // Make a new input queue that we will put the chunk's input data into. @@ -415,6 +408,7 @@ std::uint64_t asyncCompressChunks( }); // Pass the output queue to the writer thread. chunks.push(std::move(out)); + state.log(VERBOSE, "Starting a new frame\n"); // Fill the input queue for the compression job we just started status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead); } @@ -551,11 +545,14 @@ std::uint64_t asyncDecompressFrames( if (frameSize == 0) { // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted // Pass the rest of the source to this decompression task + state.log(VERBOSE, + "Input not in pzstd format, falling back to serial decompression\n"); while (status == FileStatus::Continue && !state.errorHolder.hasError()) { status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead); } break; } + state.log(VERBOSE, "Decompressing a frame of size %zu", frameSize); // Fill the input queue for the decompression job we just started status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead); } diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index 9fb2c488..dc60dd9b 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -40,7 +40,8 @@ class SharedState { if (!options.decompress) { auto parameters = options.determineParameters(); cStreamPool.reset(new ResourcePool{ - [parameters]() -> ZSTD_CStream* { + [this, parameters]() -> ZSTD_CStream* { + this->log(VERBOSE, "Creating new ZSTD_CStream\n"); auto zcs = ZSTD_createCStream(); if (zcs) { auto err = ZSTD_initCStream_advanced( @@ -57,7 +58,8 @@ class SharedState { }}); } else { dStreamPool.reset(new ResourcePool{ - []() -> ZSTD_DStream* { + [this]() -> ZSTD_DStream* { + this->log(VERBOSE, "Creating new ZSTD_DStream\n"); auto zds = ZSTD_createDStream(); if (zds) { auto err = ZSTD_initDStream(zds); @@ -74,6 +76,12 @@ class SharedState { } } + ~SharedState() { + // The resource pools have references to this, so destroy them first. + cStreamPool.reset(); + dStreamPool.reset(); + } + Logger log; ErrorHolder errorHolder; std::unique_ptr> cStreamPool; From bcd61586a86dd3efb9b800636f6c7c83b10942cb Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 15 Nov 2016 17:46:28 -0800 Subject: [PATCH 6/8] [pzstd] Cast unused parameters to void --- contrib/pzstd/Pzstd.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 1778fef2..f4cb19d9 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -341,6 +341,8 @@ static size_t calculateStep( std::uintmax_t size, size_t numThreads, const ZSTD_parameters ¶ms) { + (void)size; + (void)numThreads; return size_t{1} << (params.cParams.windowLog + 2); } From 52afb3993ee93dbbac1fd7f732e0cdfd401f91aa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Nov 2016 08:50:54 -0800 Subject: [PATCH 7/8] zbuff API now generates deprecation warnings --- NEWS | 4 ++- lib/Makefile | 3 +- lib/README.md | 6 ++-- lib/common/zbuff.h | 73 +++++++++++++++++++++++++++++----------------- programs/zstdcli.c | 2 +- tests/Makefile | 15 ++++------ 6 files changed, 62 insertions(+), 41 deletions(-) diff --git a/NEWS b/NEWS index df23a291..e6fb2be6 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ v1.1.2 -New : cli : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd) +cli : new : preserve file attributes, by Przemyslaw Skibinski +cli : fixed : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd) +API : changed : zbuff prototypes now generate deprecation warnings v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption diff --git a/lib/Makefile b/lib/Makefile index 02eefc3f..3aacb831 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -23,7 +23,7 @@ PREFIX ?= /usr/local LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include -CPPFLAGS= -I. -I./common +CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_ CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ @@ -110,6 +110,7 @@ install: libzstd.a libzstd libzstd.pc @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed diff --git a/lib/README.md b/lib/README.md index efcbdc61..d321f06b 100644 --- a/lib/README.md +++ b/lib/README.md @@ -46,9 +46,9 @@ Other optional functionalities provided are : #### Obsolete streaming API Streaming is now provided within `zstd.h`. -Older streaming API is still provided within `common/zbuff.h`. -It is considered obsolete, and will be removed in a future version. -Consider migrating towards newer streaming API. +Older streaming API is still available within `common/zbuff.h`. +It is now deprecated, and will be removed in a future version. +Consider migrating towards newer streaming API in `zstd.h`. #### Miscellaneous diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index f99e0619..e8af504d 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -9,11 +9,11 @@ /* *************************************************************** * NOTES/WARNINGS -*****************************************************************/ -/* The streaming API defined here will soon be deprecated by the -* new one in 'zstd.h'; consider migrating towards newer streaming -* API. See 'lib/README.md'. -*****************************************************************/ +******************************************************************/ +/* The streaming API defined here is deprecated. + * Consider migrating towards ZSTD_compressStream() API in `zstd.h` + * See 'lib/README.md'. + *****************************************************************/ #ifndef ZSTD_BUFFERED_H_23987 #define ZSTD_BUFFERED_H_23987 @@ -39,6 +39,27 @@ extern "C" { # define ZSTDLIB_API #endif +/* Deprecation warnings */ +/* Should these warnings be a problem, + it is generally possible to disable them, + typically with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS +# define ZBUFF_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +# define ZBUFF_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZBUFF_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZBUFF_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler") +# define ZBUFF_DEPRECATED(message) +# endif +#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */ + /* ************************************* * Streaming functions @@ -50,15 +71,15 @@ extern "C" { * frames created by one can be decoded by the other one */ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; -ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); -ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); +ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); -ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); -ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); -ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); +ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); +ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); +ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); /*-************************************************* * Streaming compression - howto @@ -102,13 +123,13 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; -ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); -ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); +ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); -ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); -ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); +ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, +ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); @@ -141,15 +162,15 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, /* ************************************* * Tool functions ***************************************/ -ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); -ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); +ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode); +ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode); /** Functions below provide recommended buffer sizes for Compression or Decompression operations. * These sizes are just hints, they tend to offer better latency */ -ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); -ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); -ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); -ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); +ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void); +ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void); +ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void); +ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void); #ifdef ZBUFF_STATIC_LINKING_ONLY @@ -169,15 +190,15 @@ ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); /*--- Custom memory allocator ---*/ /*! ZBUFF_createCCtx_advanced() : * Create a ZBUFF compression context using external alloc and free functions */ -ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); +ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); /*! ZBUFF_createDCtx_advanced() : * Create a ZBUFF decompression context using external alloc and free functions */ -ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); +ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /*--- Advanced Streaming Initialization ---*/ -ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, +ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a4b8486b..63ee99da 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -21,7 +21,7 @@ /*-************************************ -* Includes +* Dependencies **************************************/ #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include /* strcmp, strlen */ diff --git a/tests/Makefile b/tests/Makefile index c931adbe..d3a280e5 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -14,10 +14,8 @@ # paramgrill : parameter tester for zstd # test-zstd-speed.py : script for testing zstd speed difference between commits # versionsTest : compatibility test between zstd versions stored on Github (v0.1+) -# zbufftest : Test tool, to check ZBUFF integrity on target platform -# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode # zstreamtest : Fuzzer test tool for zstd streaming API -# zbufftest32: Same as zstreamtest, but forced to compile in 32-bits mode +# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode # ########################################################################## DESTDIR?= @@ -63,9 +61,9 @@ ZSTDRTTEST= --test-large-data default: fullbench -all: fullbench fuzzer zbufftest zstreamtest paramgrill datagen +all: fullbench fuzzer zstreamtest paramgrill datagen -all32: fullbench32 fuzzer32 zbufftest32 zstreamtest32 +all32: fullbench32 fuzzer32 zstreamtest32 @@ -139,7 +137,7 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr HOST_OS = POSIX valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench zbufftest +valgrindTest: zstd datagen fuzzer fullbench @echo "\n ---- valgrind tests : memory analyzer ----" $(VALGRIND) ./datagen -g50M > $(VOID) $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi @@ -151,7 +149,6 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest @rm tmp $(VALGRIND) ./fuzzer -T1mn -t1 $(VALGRIND) ./fullbench -i1 - $(VALGRIND) ./zbufftest -T1mn endif @@ -169,9 +166,9 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream +test: test-zstd test-fullbench test-fuzzer test-zstream -test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 test-all: test test32 valgrindTest From 883a7cacc4b58ca2de4ce91229f20811895c8db5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Nov 2016 08:58:32 -0800 Subject: [PATCH 8/8] removed zbufftest from list of clang tests on Appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 6c1e0bc2..f49d7cf4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,7 +25,7 @@ install: - MKDIR bin - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%COMPILER%]==[gcc] ( - SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && + SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&