AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-11-18 11:47:01 +01:00
commit 5a17223691
19 changed files with 154 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.travis.yml
View File

@ -12,12 +12,13 @@ matrix:
os: linux
sudo: false
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean"
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
os: linux
sudo: false
language: cpp
install:
- export CXX="g++-4.8" CC="gcc-4.8"
- export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
addons:
apt:
sources:
@ -25,11 +26,11 @@ matrix:
packages:
- gcc-4.8
- g++-4.8
- env: Ubu=12.04cont Cmd="make usan"
os: linux
sudo: false
- env: Ubu=12.04cont Cmd="make asan"
os: linux
sudo: false
@ -39,7 +40,7 @@ matrix:
- env: Ubu=12.04 Cmd="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make -C programs clean && make -C tests versionsTest"
os: linux
sudo: required
- env: Ubu=12.04 Cmd="make asan32"
os: linux
sudo: required
@ -50,7 +51,7 @@ matrix:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..'
os: linux
sudo: required
@ -65,8 +66,8 @@ matrix:
packages:
- gcc-6
- g++-6
# Ubuntu 14.04 LTS Server Edition 64 bit
- env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
dist: trusty
@ -99,7 +100,7 @@ matrix:
apt:
packages:
- valgrind
- env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
os: linux
dist: trusty
@ -115,7 +116,7 @@ matrix:
- gcc-4.8-multilib
- g++-4.8
- g++-4.8-multilib
- env: Ubu=14.04 Cmd="make -C tests test32"
os: linux
dist: trusty
@ -125,7 +126,7 @@ matrix:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
os: linux
dist: trusty

4
NEWS
View File

@ -1,5 +1,7 @@
v1.1.2
New : cli : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd)
cli : new : preserve file attributes, by Przemyslaw Skibinski
cli : fixed : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd)
API : changed : zbuff prototypes now generate deprecation warnings
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption

5
appveyor.yml
View File

@ -25,7 +25,7 @@ install:
- MKDIR bin
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
- if [%COMPILER%]==[gcc] (
SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&
@ -51,7 +51,8 @@ build_script:
ECHO *** Building pzstd for %PLATFORM% &&
ECHO *** &&
make -C contrib\pzstd googletest-mingw64 &&
make -C contrib\pzstd all &&
make -C contrib\pzstd pzstd.exe &&
make -C contrib\pzstd tests &&
make -C contrib\pzstd check &&
make -C contrib\pzstd clean
)

18
contrib/pzstd/Makefile
View File

@ -34,7 +34,7 @@ LDFLAGS ?=
PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
GTEST_INC = -isystem googletest/googletest/include
PZSTD_CPPFLAGS = $(PZSTD_INC) $(GTEST_INC)
PZSTD_CPPFLAGS = $(PZSTD_INC)
PZSTD_CCXXFLAGS =
PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS)
PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS)
@ -47,10 +47,10 @@ ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
# gtest libraries need to go before "-lpthread" because they depend on it.
GTEST_LIB = -L googletest/build/googlemock/gtest
LIBS = $(GTEST_LIB) -lpthread
LIBS =
# Compilation commands
LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -o $@
LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -lpthread -o $@
CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@
CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@
@ -109,7 +109,7 @@ uninstall:
# Targets for many different builds
.PHONY: all
all: PZSTD_CPPFLAGS += -DNDEBUG
all: pzstd$(EXT) tests roundtrip
all: pzstd$(EXT)
.PHONY: debug
debug: EXTRA_FLAGS += -g
@ -130,7 +130,7 @@ ubsan: debug
.PHONY: all32
all32: EXTRA_FLAGS += -m32
all32: all
all32: all tests roundtrip
.PHONY: debug32
debug32: EXTRA_FLAGS += -m32
@ -177,12 +177,14 @@ test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \
Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND)
test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main
test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
test/%Test$(EXT): LIBS += -lgtest -lgtest_main
test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \
SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND)
utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main
utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
utils/test/%Test$(EXT): LIBS += -lgtest -lgtest_main
utils/test/%Test$(EXT): utils/test/%Test.o
$(LD_COMMAND)
@ -233,10 +235,12 @@ $(PROGDIR)/%.o: $(PROGDIR)/%.c
$(CXX_COMMAND)
$(POSTCOMPILE)
test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
test/%.o: test/%.cpp
$(CXX_COMMAND)
$(POSTCOMPILE)
utils/test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
utils/test/%.o: utils/test/%.cpp
$(CXX_COMMAND)
$(POSTCOMPILE)

17
contrib/pzstd/Pzstd.cpp
View File

@ -341,15 +341,9 @@ static size_t calculateStep(
std::uintmax_t size,
size_t numThreads,
const ZSTD_parameters &params) {
size_t step = size_t{1} << (params.cParams.windowLog + 2);
// If file size is known, see if a smaller step will spread work more evenly
if (size != 0) {
const std::uintmax_t newStep = size / numThreads;
if (newStep != 0 && newStep <= std::numeric_limits<size_t>::max()) {
step = std::min(step, static_cast<size_t>(newStep));
}
}
return step;
(void)size;
(void)numThreads;
return size_t{1} << (params.cParams.windowLog + 2);
}
namespace {
@ -401,6 +395,7 @@ std::uint64_t asyncCompressChunks(
// Break the input up into chunks of size `step` and compress each chunk
// independently.
size_t step = calculateStep(size, numThreads, params);
state.log(DEBUG, "Chosen frame size: %zu\n", step);
auto status = FileStatus::Continue;
while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
// Make a new input queue that we will put the chunk's input data into.
@ -415,6 +410,7 @@ std::uint64_t asyncCompressChunks(
});
// Pass the output queue to the writer thread.
chunks.push(std::move(out));
state.log(VERBOSE, "Starting a new frame\n");
// Fill the input queue for the compression job we just started
status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
}
@ -551,11 +547,14 @@ std::uint64_t asyncDecompressFrames(
if (frameSize == 0) {
// We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
// Pass the rest of the source to this decompression task
state.log(VERBOSE,
"Input not in pzstd format, falling back to serial decompression\n");
while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead);
}
break;
}
state.log(VERBOSE, "Decompressing a frame of size %zu", frameSize);
// Fill the input queue for the decompression job we just started
status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead);
}

12
contrib/pzstd/Pzstd.h
View File

@ -40,7 +40,8 @@ class SharedState {
if (!options.decompress) {
auto parameters = options.determineParameters();
cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
[parameters]() -> ZSTD_CStream* {
[this, parameters]() -> ZSTD_CStream* {
this->log(VERBOSE, "Creating new ZSTD_CStream\n");
auto zcs = ZSTD_createCStream();
if (zcs) {
auto err = ZSTD_initCStream_advanced(
@ -57,7 +58,8 @@ class SharedState {
}});
} else {
dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
[]() -> ZSTD_DStream* {
[this]() -> ZSTD_DStream* {
this->log(VERBOSE, "Creating new ZSTD_DStream\n");
auto zds = ZSTD_createDStream();
if (zds) {
auto err = ZSTD_initDStream(zds);
@ -74,6 +76,12 @@ class SharedState {
}
}
~SharedState() {
// The resource pools have references to this, so destroy them first.
cStreamPool.reset();
dStreamPool.reset();
}
Logger log;
ErrorHolder errorHolder;
std::unique_ptr<ResourcePool<ZSTD_CStream>> cStreamPool;

5
contrib/pzstd/README.md
View File

@ -51,5 +51,6 @@ Pigz cannot do parallel decompression, it simply does each of reading, decompres
## Tests
Tests require that you have [gtest](https://github.com/google/googletest) installed.
Modify `GTEST_INC` and `GTEST_LIB` in `test/Makefile` and `utils/test/Makefile` to work for your install of gtest.
Then run `make test` in the `contrib/pzstd` directory.
Set `GTEST_INC` and `GTEST_LIB` in `Makefile` to specify the location of the gtest headers and libraries.
Alternatively, run `make googletest`, which will clone googletest and build it.
Run `make tests && make check` to run tests.

3
lib/Makefile
View File

@ -23,7 +23,7 @@ PREFIX ?= /usr/local
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
CPPFLAGS= -I. -I./common
CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
@ -110,6 +110,7 @@ install: libzstd.a libzstd libzstd.pc
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed

6
lib/README.md
View File

@ -46,9 +46,9 @@ Other optional functionalities provided are :
#### Obsolete streaming API
Streaming is now provided within `zstd.h`.
Older streaming API is still provided within `common/zbuff.h`.
It is considered obsolete, and will be removed in a future version.
Consider migrating towards newer streaming API.
Older streaming API is still available within `common/zbuff.h`.
It is now deprecated, and will be removed in a future version.
Consider migrating towards newer streaming API in `zstd.h`.
#### Miscellaneous

73
lib/common/zbuff.h
View File

@ -9,11 +9,11 @@
/* ***************************************************************
* NOTES/WARNINGS
*****************************************************************/
/* The streaming API defined here will soon be deprecated by the
* new one in 'zstd.h'; consider migrating towards newer streaming
* API. See 'lib/README.md'.
*****************************************************************/
******************************************************************/
/* The streaming API defined here is deprecated.
* Consider migrating towards ZSTD_compressStream() API in `zstd.h`
* See 'lib/README.md'.
*****************************************************************/
#ifndef ZSTD_BUFFERED_H_23987
#define ZSTD_BUFFERED_H_23987
@ -39,6 +39,27 @@ extern "C" {
# define ZSTDLIB_API
#endif
/* Deprecation warnings */
/* Should these warnings be a problem,
it is generally possible to disable them,
typically with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual.
Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
# define ZBUFF_DEPRECATED(message) /* disable deprecation warnings */
#else
# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
# define ZBUFF_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZBUFF_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define ZBUFF_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler")
# define ZBUFF_DEPRECATED(message)
# endif
#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */
/* *************************************
* Streaming functions
@ -50,15 +71,15 @@ extern "C" {
* frames created by one can be decoded by the other one */
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
/*-*************************************************
* Streaming compression - howto
@ -102,13 +123,13 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
@ -141,15 +162,15 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
/* *************************************
* Tool functions
***************************************/
ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode);
ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void);
ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void);
ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
#ifdef ZBUFF_STATIC_LINKING_ONLY
@ -169,15 +190,15 @@ ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
/*--- Custom memory allocator ---*/
/*! ZBUFF_createCCtx_advanced() :
* Create a ZBUFF compression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZBUFF_createDCtx_advanced() :
* Create a ZBUFF decompression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
/*--- Advanced Streaming Initialization ---*/
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);

22
lib/decompress/zstd_decompress.c
View File

@ -104,7 +104,6 @@ struct ZSTD_DCtx_s
U32 dictID;
const BYTE* litPtr;
ZSTD_customMem customMem;
size_t litBufSize;
size_t litSize;
size_t rleSize;
BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
@ -429,10 +428,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
@ -459,13 +458,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@ -490,9 +488,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break;
}
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@ -867,7 +864,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@ -879,7 +876,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
/* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */
/* copy literals */
@ -914,7 +911,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@ -926,8 +923,8 @@ size_t ZSTD_execSequence(BYTE* op,
/* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd);
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */
ZSTD_copy8(op, *litPtr);
@ -1002,7 +999,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
@ -1028,7 +1024,7 @@ static size_t ZSTD_decompressSequences(
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }

14
lib/legacy/zstd_v02.c
View File

@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s
blockType_t bType;
U32 phase;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */
}
case IS_RAW:
@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+3;
dctx->litBufSize = srcSize-3;
dctx->litSize = litSize;
return litSize+3;
}
@ -2966,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
{
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[3], litSize);
memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
return 4;
}
@ -3175,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* checks */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
/* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@ -3241,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litMax = litPtr + dctx->litBufSize;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@ -3277,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}

14
lib/legacy/zstd_v03.c
View File

@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s
blockType_t bType;
U32 phase;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */
}
case IS_RAW:
@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+3;
dctx->litBufSize = srcSize-3;
dctx->litSize = litSize;
return litSize+3;
}
@ -2607,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
{
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[3], litSize);
memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
return 4;
}
@ -2816,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* checks */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
/* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@ -2882,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litMax = litPtr + dctx->litBufSize;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@ -2918,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}

16
lib/legacy/zstd_v04.c
View File

@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s
blockType_t bType;
ZSTD_dStage stage;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */
}
case IS_RAW:
@ -2859,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+3;
dctx->litBufSize = srcSize-3;
dctx->litSize = litSize;
return litSize+3; }
case IS_RLE:
{
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[3], litSize);
memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
return 4;
}
@ -3069,7 +3066,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@ -3084,7 +3081,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@ -3167,7 +3164,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@ -3206,7 +3202,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}

18
lib/legacy/zstd_v05.c
View File

@ -2731,7 +2731,6 @@ struct ZSTDv05_DCtx_s
ZSTDv05_dStage stage;
U32 flagStaticTables;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
@ -2978,8 +2977,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_PCH:
@ -3002,8 +3001,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_RAW:
@ -3028,13 +3027,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@ -3057,9 +3055,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break;
}
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@ -3289,7 +3286,7 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTDv05_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@ -3304,7 +3301,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
/* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */
ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@ -3378,7 +3375,6 @@ static size_t ZSTDv05_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@ -3416,7 +3412,7 @@ static size_t ZSTDv05_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTDv05_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}

18
lib/legacy/zstd_v06.c
View File

@ -2893,7 +2893,6 @@ struct ZSTDv06_DCtx_s
ZSTDv06_dStage stage;
U32 flagRepeatTable;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[ZSTDv06_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
@ -3170,8 +3169,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_PCH:
@ -3192,8 +3191,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
}
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_RAW:
@ -3217,13 +3216,12 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@ -3245,9 +3243,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
break;
}
if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@ -3438,7 +3435,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
size_t ZSTDv06_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@ -3451,7 +3448,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
/* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
/* copy Literals */
ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@ -3523,7 +3520,6 @@ static size_t ZSTDv06_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
FSEv06_DTable* DTableLL = dctx->LLTable;
FSEv06_DTable* DTableML = dctx->MLTable;
@ -3567,7 +3563,7 @@ static size_t ZSTDv06_decompressSequences(
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
#endif
{ size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
{ size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv06_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }

18
lib/legacy/zstd_v07.c
View File

@ -3021,7 +3021,6 @@ struct ZSTDv07_DCtx_s
U32 dictID;
const BYTE* litPtr;
ZSTDv07_customMem customMem;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX];
@ -3395,9 +3394,9 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize;
dctx->litEntropy = 1;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case lbt_repeat:
@ -3418,8 +3417,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
if (HUFv07_isError(errorCode)) return ERROR(corruption_detected);
}
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case lbt_raw:
@ -3443,13 +3442,12 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@ -3471,9 +3469,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
break;
}
if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@ -3662,7 +3659,7 @@ static seq_t ZSTDv07_decodeSequence(seqState_t* seqState)
static
size_t ZSTDv07_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@ -3674,7 +3671,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
/* check */
if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
/* copy Literals */
ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
@ -3746,7 +3743,6 @@ static size_t ZSTDv07_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize;
FSEv07_DTable* DTableLL = dctx->LLTable;
FSEv07_DTable* DTableML = dctx->MLTable;
@ -3776,7 +3772,7 @@ static size_t ZSTDv07_decompressSequences(
for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) {
nbSeq--;
{ seq_t const sequence = ZSTDv07_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }

2
programs/zstdcli.c
View File

@ -21,7 +21,7 @@
/*-************************************
* Includes
* Dependencies
**************************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include <string.h> /* strcmp, strlen */

15
tests/Makefile
View File

@ -14,10 +14,8 @@
# paramgrill : parameter tester for zstd
# test-zstd-speed.py : script for testing zstd speed difference between commits
# versionsTest : compatibility test between zstd versions stored on Github (v0.1+)
# zbufftest : Test tool, to check ZBUFF integrity on target platform
# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode
# zstreamtest : Fuzzer test tool for zstd streaming API
# zbufftest32: Same as zstreamtest, but forced to compile in 32-bits mode
# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
# ##########################################################################
DESTDIR?=
@ -63,9 +61,9 @@ ZSTDRTTEST= --test-large-data
default: fullbench
all: fullbench fuzzer zbufftest zstreamtest paramgrill datagen
all: fullbench fuzzer zstreamtest paramgrill datagen
all32: fullbench32 fuzzer32 zbufftest32 zstreamtest32
all32: fullbench32 fuzzer32 zstreamtest32
@ -139,7 +137,7 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
HOST_OS = POSIX
valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
valgrindTest: zstd datagen fuzzer fullbench zbufftest
valgrindTest: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
$(VALGRIND) ./datagen -g50M > $(VOID)
$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
@ -151,7 +149,6 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest
@rm tmp
$(VALGRIND) ./fuzzer -T1mn -t1
$(VALGRIND) ./fullbench -i1
$(VALGRIND) ./zbufftest -T1mn
endif
@ -169,9 +166,9 @@ zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
test: test-zstd test-fullbench test-fuzzer test-zstream
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
test-all: test test32 valgrindTest