diff --git a/appveyor.yml b/appveyor.yml index b5ffb4a2..9a493a61 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,11 @@ version: 1.0.{build} environment: matrix: - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw32" + - ZSTD_APPVEYOR: "gcc" + MAKE_PARAMS: "test" + PLATFORM: "mingw64" - ZSTD_APPVEYOR: "visual" CONFIGURATION: "Debug" PLATFORM: "Win32" @@ -18,7 +23,6 @@ environment: install: - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%ZSTD_APPVEYOR%]==[gcc] ( - SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" && SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && @@ -29,24 +33,9 @@ install: build_script: - ECHO Building %ZSTD_APPVEYOR% - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw32 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe && - make clean - ) - - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%ZSTD_APPVEYOR%]==[gcc] ( - ECHO *** && - ECHO *** Building mingw64 && - ECHO *** && - ECHO make %MINGW_PARAMS% && - make %MINGW_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe && + - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] ( make clean && ECHO *** && ECHO *** Building clang && @@ -56,6 +45,15 @@ build_script: COPY programs\fuzzer.exe projects\fuzzer_clang.exe && make clean ) + - if [%ZSTD_APPVEYOR%]==[gcc] ( + ECHO *** && + ECHO *** Building %PLATFORM% && + ECHO *** && + ECHO make %MAKE_PARAMS% && + make %MAKE_PARAMS% && + COPY programs\fuzzer.exe projects\fuzzer_%PLATFORM%.exe && + make clean + ) - if [%ZSTD_APPVEYOR%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && @@ -82,14 +80,14 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\zstd.exe projects\zstd_VS2015_%PLATFORM%_%CONFIGURATION%.exe ) test_script: - SET FUZZERTEST=-T1mn - if [%ZSTD_APPVEYOR%]==[gcc] ( - projects\fuzzer_mingw32.exe %FUZZERTEST% && - projects\fuzzer_mingw64.exe %FUZZERTEST% && - projects\fuzzer_clang.exe %FUZZERTEST% + projects\fuzzer_%PLATFORM%.exe %FUZZERTEST% && + if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% ) - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] ( projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && diff --git a/programs/Makefile b/programs/Makefile index 70434ced..339427e2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -163,7 +163,11 @@ clean: #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) +HOST_OS = POSIX +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) install: zstd @echo Installing binaries @@ -186,15 +190,41 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: zstd datagen fuzzer fullbench zbufftest + @echo "\n ---- valgrind tests : memory analyzer ----" + $(VALGRIND) ./datagen -g50M > $(VOID) + $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) + ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp + $(VALGRIND) ./zstd -vdf tmp -o $(VOID) + ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) + @rm tmp + $(VALGRIND) ./fuzzer -T1mn -t1 + $(VALGRIND) ./fullbench -i1 + $(VALGRIND) ./zbufftest -T1mn + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +zstd-playTests: datagen + ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST) +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) test: test-zstd test-fullbench test-fuzzer test-zbuff test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest -zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) - test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests @@ -223,20 +253,4 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) - -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench zbufftest - @echo "\n ---- valgrind tests : memory analyzer ----" - $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) - ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp - $(VALGRIND) ./zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) - @rm tmp - $(VALGRIND) ./fuzzer -T1mn -t1 - $(VALGRIND) ./fullbench -i1 - $(VALGRIND) ./zbufftest -T1mn - endif diff --git a/programs/playTests.sh b/programs/tests/playTests.sh old mode 100755 new mode 100644 similarity index 100% rename from programs/playTests.sh rename to programs/tests/playTests.sh diff --git a/programs/tests/playTestsMSYS.sh b/programs/tests/playTestsMSYS.sh new file mode 100644 index 00000000..82f2357b --- /dev/null +++ b/programs/tests/playTestsMSYS.sh @@ -0,0 +1,160 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + fc tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + + +echo "\n**** simple tests **** " +./datagen > tmp +$ZSTD -f tmp # trivial compression case, creates tmp.zst +$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) +echo "test : too large compression level (must fail)" +$ZSTD -99 tmp && die "too large compression level undetected" +echo "test : compress to stdout" +$ZSTD tmp -c > tmpCompressed +$ZSTD tmp --stdout > tmpCompressed # long command format +echo "test : null-length file roundtrip" +echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout +echo "test : decompress file with wrong suffix (must fail)" +$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" +$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout +$ZSTD --decompress tmpCompressed -c > tmpResult +$ZSTD --decompress tmpCompressed --stdout > tmpResult +#$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout +#$ZSTD -d - < tmp.zst > /dev/null +$ZSTD -dc < tmp.zst > /dev/null +$ZSTD -dc - < tmp.zst > /dev/null +$ZSTD -q tmp && die "overwrite check failed!" +$ZSTD -q -f tmp +$ZSTD -q --force tmp +$ZSTD -df tmp && die "should have refused : wrong extension" +cp tmp tmp2.zst +$ZSTD -df tmp2.zst && die "should have failed : wrong format" +rm tmp2.zst + +echo "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD -c hello.tmp > hello.zstd +$ZSTD -c world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -dc helloworld.zstd > result.tmp +cat result.tmp +fc helloworld.tmp result.tmp +rm ./*.tmp ./*.zstd + +echo frame concatenation test completed + + +echo "\n**** dictionary tests **** " + +./datagen > tmpDict +./datagen -g1M | md5sum > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +fc tmp1 tmp2 +$ZSTD --train *.c *.h -o tmpDict +$ZSTD xxhash.c -D tmpDict -of tmp +$ZSTD -d tmp -D tmpDict -of result +fc xxhash.c result + + +echo "\n**** multiple files tests **** " + +./datagen -s1 > tmp1 2> /dev/null +./datagen -s2 -g100K > tmp2 2> /dev/null +./datagen -s3 -g1M > tmp3 2> /dev/null +$ZSTD -f tmp* +echo "compress tmp* : " +ls -ls tmp* +rm tmp1 tmp2 tmp3 +echo "decompress tmp* : " +$ZSTD -df *.zst +ls -ls tmp* +echo "compress tmp* into stdout > tmpall : " +$ZSTD -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* +echo "decompress tmpall* into stdout > tmpdec : " +cp tmpall tmpall2 +# $ZSTD -dc tmpall* > tmpdec +ls -ls tmp* +echo "compress multiple files including a missing one (notHere) : " +$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" + +echo "\n**** integrity tests **** " +echo "test one file (tmp1.zst) " +$ZSTD -t tmp1.zst +$ZSTD --test tmp1.zst +echo "test multiple files (*.zst) " +$ZSTD -t *.zst +echo "test good and bad files (*) " +$ZSTD -t * && die "bad files not detected !" + +echo "\n**** zstd round-trip tests **** " + +roundTripTest +roundTripTest -g15K # TableID==3 +roundTripTest -g127K # TableID==2 +roundTripTest -g255K # TableID==1 +roundTripTest -g513K # TableID==0 +roundTripTest -g512K 6 # greedy, hash chain +roundTripTest -g512K 16 # btlazy2 +roundTripTest -g512K 19 # btopt + +rm tmp* + +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 + +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 1 + +rm tmp* +