AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'dev' into lowAddr

Browse Source
This commit is contained in:
Yann Collet 2018-04-16 16:12:38 -07:00
commit b5528ac86c
4 changed files with 115 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -143,10 +143,10 @@ clangtest-native: clean
@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
usan: clean
CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
usan32: clean
CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
staticAnalyze: clean
CFLAGS=-g scan-build --status-bugs -v $(MAKE) all

14
circle.yml
View File

@ -11,10 +11,10 @@ test:
- clang -v; make clangtest && make clean
- g++ -v; make gpptest && make clean
- gcc -v; make c_standards && make clean
- gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean
- gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean
- gcc-6 -v; make c_standards CC=gcc-6 && make clean
- gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
- gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
- gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- gcc-6 -v; CC=gcc-6 make c_standards && make clean
- gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean
# Shorter tests
- make cmake && make clean
- make -C tests test-lz4
@ -22,11 +22,11 @@ test:
- make -C tests test-frametest
- make -C tests test-fullbench
- make -C tests test-fuzzer && make clean
- make -C lib all && make clean
- pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
- make -C lib all && make clean
- pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
- make travis-install && make clean
# Longer tests
- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
- gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- make usan && make clean
- clang -v; make staticAnalyze && make clean
# Valgrind tests

15
tests/Makefile
View File

@ -40,7 +40,7 @@ CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
@ -57,7 +57,7 @@ LZ4 := $(PRGDIR)/lz4$(EXT)
# Default test parameters
TEST_FILES := COPYING
FUZZER_TIME := -T3mn
FUZZER_TIME := -T90s
NB_LOOPS ?= -i1
@ -394,13 +394,12 @@ test-mem: lz4 datagen fuzzer frametest fullbench
./datagen -g16KB -s2 > ftmdg16K2
./datagen -g16KB -s3 > ftmdg16K3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
./datagen -g16MB > ftmdg16M
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2
./datagen -g7MB > ftmdg7M
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2
./datagen -g256MB > ftmdg256M
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID)
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
$(RM) ftm*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256

101
tests/fuzzer.c
View File

@ -47,6 +47,7 @@
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#define LZ4_STATIC_LINKING_ONLY
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
#define XXH_STATIC_LINKING_ONLY
@ -350,9 +351,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
U32 testNb = 0;
U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
int const dictSize = MIN(dictSizeRand, blockStart);
int const dictSize = MIN(dictSizeRand, blockStart - 1);
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
const char* block = ((char*)CNBuffer) + blockStart;
const char* dict = block - dictSize;
@ -452,6 +453,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
/* Test compression using fast reset external state*/
FUZ_DISPLAYTEST;
ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
/* Test compression */
FUZ_DISPLAYTEST("test LZ4_compress_default()");
ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
@ -680,6 +686,97 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
} }
/* Compress using external dictionary stream */
FUZ_DISPLAYTEST;
{
LZ4_stream_t LZ4_stream;
int expectedSize;
U32 expectedCrc;
LZ4_loadDict(&LZ4dict, dict, dictSize);
expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
/* In the future, it might be desirable to let extDictCtx mode's
* output diverge from the output generated by regular extDict mode.
* Until that time, this comparison serves as a good regression
* test.
*/
FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
FUZ_DISPLAYTEST;
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST;
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
FUZ_DISPLAYTEST;
LZ4_resetStream_fast(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
}
/* Decompress with dictionary as external */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
{ U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes) {
decodedBuffer[blockSize-missingBytes] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
} }
/* Compress HC using External dictionary */
FUZ_DISPLAYTEST();
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */