From 106278e7e5fafaea3b7deb4147bdc8071562d2f0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 12 Nov 2019 18:59:06 -0500 Subject: [PATCH 01/10] Use lowercase windows.h Helps for cross builds, doesn't matter on Windows itself. --- programs/timefn.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/timefn.h b/programs/timefn.h index d1ddd31b..37802a80 100644 --- a/programs/timefn.h +++ b/programs/timefn.h @@ -47,7 +47,7 @@ extern "C" { ******************************************/ #if defined(_WIN32) /* Windows */ - #include /* LARGE_INTEGER */ + #include /* LARGE_INTEGER */ typedef LARGE_INTEGER UTIL_time_t; #define UTIL_TIME_INITIALIZER { { 0, 0 } } From 0ede342acc2c26f87ae962fa88e158904d4198c4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 12 Nov 2019 15:13:35 -0500 Subject: [PATCH 02/10] Fix MinGW cross build from Unix generate_res.bat seems rather pointless, am I missing something? I just inlined it into the Makefile. --- programs/Makefile | 26 +++++++++++++++++--------- programs/windres/generate_res.bat | 11 ----------- 2 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 programs/windres/generate_res.bat diff --git a/programs/Makefile b/programs/Makefile index 64dcae00..5ed61f97 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -165,15 +165,15 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +ifneq (,$(filter Windows%,$(OS))) +zstd : $(RES_FILE) +endif zstd : $(ZSTDLIB_FILES) zstdcli.o util.o timefn.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" @echo "$(LZ4_MSG)" -ifneq (,$(filter Windows%,$(OS))) - windres/generate_res.bat -endif - $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) + $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) .PHONY: zstd-release zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0 @@ -183,11 +183,11 @@ zstd-release: zstd zstd32 : CPPFLAGS += $(THREAD_CPP) zstd32 : LDFLAGS += $(THREAD_LD) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c timefn.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c ifneq (,$(filter Windows%,$(OS))) - windres/generate_res.bat +zstd32 : $(RES32_FILE) endif - $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) +zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c timefn.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o timefn.o datagen.o dibio.o $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) @@ -235,8 +235,16 @@ zstdmt: zstd ln -sf zstd zstdmt .PHONY: generate_res -generate_res: - windres/generate_res.bat +generate_res: $(RES64_FILE) $(RES32_FILE) + +ifneq (,$(filter Windows%,$(OS))) +RC ?= windres +# http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable +$(RES64_FILE): windres/zstd.rc + $(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64 +$(RES32_FILE): windres/zstd.rc + $(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-i386 +endif .PHONY: clean clean: diff --git a/programs/windres/generate_res.bat b/programs/windres/generate_res.bat deleted file mode 100644 index 7ff9aef5..00000000 --- a/programs/windres/generate_res.bat +++ /dev/null @@ -1,11 +0,0 @@ -@echo off -REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable - -where /q windres.exe -IF ERRORLEVEL 1 ( - ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH. - EXIT /B -) ELSE ( - windres.exe -I ../lib -I windres -i windres/zstd.rc -O coff -F pe-x86-64 -o windres/zstd64.res - windres.exe -I ../lib -I windres -i windres/zstd.rc -O coff -F pe-i386 -o windres/zstd32.res -) From 10552eaffef84c011f67af0e04f0780b50a5ab26 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 12 Nov 2019 15:34:00 -0500 Subject: [PATCH 03/10] Fix MinGW cross install from Unix Needed a bunch of `$(EXT)` --- programs/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 5ed61f97..668e6135 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -350,10 +350,10 @@ INSTALL_MAN ?= $(INSTALL_DATA) install: zstd @echo Installing binaries @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/ - @$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd - @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat - @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd - @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdmt + @$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT) + @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT) + @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT) @$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless @$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep @echo Installing man pages From 228a6d28cd107dd0accc86dd16a1b8d5f177fd59 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Nov 2019 10:24:09 -0800 Subject: [PATCH 04/10] travis : added test durations as a guide when doing test selection --- .travis.yml | 56 ++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index ec1adab0..01859604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,39 +26,39 @@ env: matrix: fast_finish: true include: - - name: arm64 + - name: arm64 # ~2.5 mn os: linux arch: arm64 script: - make check - - name: make test (complete) + - name: make test (complete) # ~14mn script: # DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null - DEVNULLRIGHTS=test make test - - name: gcc-6 + gcc-7 compilation + - name: gcc-6 + gcc-7 compilation # ~ 3mn script: - make gcc6install gcc7install - CC=gcc-6 CFLAGS=-Werror make -j all - make clean - CC=gcc-7 CFLAGS=-Werror make -j all - - name: gcc-8 + ASan + UBSan + Test Zstd + - name: gcc-8 + ASan + UBSan + Test Zstd # ~6.5mn script: - make gcc8install - CC=gcc-8 CFLAGS="-Werror" make -j all - make clean - CC=gcc-8 make -j uasan-test-zstd Date: Tue, 26 Nov 2019 10:28:21 -0800 Subject: [PATCH 05/10] moved meson and versions compatibility tests to master/cron also : joined c99 compatibility tests with gnu90 and c++ ones --- .travis.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01859604..0b16f9dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ matrix: - make clean - make -j check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - - name: cmake test # ~2mn + - name: cmake build check # ~2mn script: - make cmakebuild @@ -122,7 +122,7 @@ matrix: - make arminstall - make armfuzz - - name: Xenial (AARCH64 + Fuzz Test) # ~14mn + - name: aarch64 + Fuzz Test (on Xenial) # ~14mn dist: xenial script: - make arminstall @@ -133,10 +133,6 @@ matrix: - make ppcinstall - make ppcfuzz - - name: Versions Compatibility Test # 11.5mn - script: - - make -C tests versionsTest - # check release number (release only) - name: Tag-Specific Test if: tag =~ ^v[0-9]\.[0-9] @@ -146,11 +142,15 @@ matrix: # tests for master branch and cron job only - name: OS-X # ~13mn - if: branch = master os: osx script: - make test + - name: Versions Compatibility Test # 11.5mn + if: branch = master + script: + - make -C tests versionsTest + - name: thread sanitizer # ~29mn if: branch = master script: @@ -158,20 +158,16 @@ matrix: - CC=clang-3.8 make tsan-test-zstream - CC=clang-3.8 make tsan-fuzztest - - name: C++ and gnu90 compatibility # ~2.5mn + - name: C++, gnu90 and c99 compatibility # ~3mn if: branch = master script: - make cxxtest - make clean - make gnu90build - make clean - - make travis-install # just ensures `make install` works - - - name: c99 compatibility # ~1.5mn - if: branch = master - script: - make c99build - - make -C tests test-zstd + - make clean + - make travis-install # just ensures `make install` works - name: PPC64 # ~13mn if: branch = master From 87ae7b940b817add132f5349d2006263d96378fd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Nov 2019 10:30:11 -0800 Subject: [PATCH 06/10] travisCI: moved compatibility tests to `dev` they are fast enough, and we want to catch such incompatibility early --- .travis.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b16f9dd..27479517 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,6 +110,16 @@ matrix: - make clean - make -j msanregressiontest + - name: C++, gnu90 and c99 compatibility # ~3mn + script: + - make cxxtest + - make clean + - make gnu90build + - make clean + - make c99build + - make clean + - make travis-install # just ensures `make install` works + - name: Valgrind + Fuzz Test Stack Mode # ~ 7mn script: - make valgrindinstall @@ -122,7 +132,7 @@ matrix: - make arminstall - make armfuzz - - name: aarch64 + Fuzz Test (on Xenial) # ~14mn + - name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn dist: xenial script: - make arminstall @@ -158,17 +168,6 @@ matrix: - CC=clang-3.8 make tsan-test-zstream - CC=clang-3.8 make tsan-fuzztest - - name: C++, gnu90 and c99 compatibility # ~3mn - if: branch = master - script: - - make cxxtest - - make clean - - make gnu90build - - make clean - - make c99build - - make clean - - make travis-install # just ensures `make install` works - - name: PPC64 # ~13mn if: branch = master script: From f57be339ab78c0a23b75eba23bbd4f4b553341e7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Nov 2019 15:01:16 -0800 Subject: [PATCH 07/10] travisCI : removed `trusty` generic statement trusty is out of long term support. let's run (most) tests on "current" platform. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27479517..524d183f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ # Medium Tests: Run on all commits/PRs to dev branch language: c -dist: trusty git: depth: 1 From ecaf3ce1031f54bc1a8605e75d7fc7f20d4f6dc5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Nov 2019 16:44:56 -0800 Subject: [PATCH 08/10] reverted ppc qemu and scanbuild on trusty ppc cross-compilation seems broken outside of trusty scan-build version changes with distrib version, resulting in a different list of false positives --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 524d183f..fd486af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,6 +85,7 @@ matrix: - make cmakebuild - name: static analyzer scanbuild # ~26mn + dist: trusty # note : it's important to pin down a version of static analyzer, since newer versions find different false positives script: - make staticAnalyze @@ -137,7 +138,8 @@ matrix: - make arminstall - make aarch64fuzz - - name: PPC + Fuzz Test # ~13mn + - name: Qemu PPC + Fuzz Test # ~13mn + dist: trusty # it seems ppc cross-compilation fails on "current" script: - make ppcinstall - make ppcfuzz From d6e32afa0ab24aee51c3989528281adc4ffad266 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Nov 2019 12:44:02 -0800 Subject: [PATCH 09/10] added mingw cross compilation tests based on a script in RocksDB project. also : pushed qemu aarch64 tests to master/cron, since there are already hardware aarch64 tests on dev. --- .travis.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd486af6..c7d027f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,7 @@ matrix: - make cmakebuild - name: static analyzer scanbuild # ~26mn - dist: trusty # note : it's important to pin down a version of static analyzer, since newer versions find different false positives + dist: trusty # note : it's important to pin down a version of static analyzer, since newer versions find different false positives script: - make staticAnalyze @@ -120,6 +120,11 @@ matrix: - make clean - make travis-install # just ensures `make install` works + - name: mingw cross-compilation + script : + - sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ make + - name: Valgrind + Fuzz Test Stack Mode # ~ 7mn script: - make valgrindinstall @@ -132,12 +137,6 @@ matrix: - make arminstall - make armfuzz - - name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn - dist: xenial - script: - - make arminstall - - make aarch64fuzz - - name: Qemu PPC + Fuzz Test # ~13mn dist: trusty # it seems ppc cross-compilation fails on "current" script: @@ -175,6 +174,14 @@ matrix: - make ppcinstall - make ppc64fuzz + # note : we already have aarch64 tests on hardware + - name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn + if: branch = master + dist: xenial + script: + - make arminstall + - make aarch64fuzz + - name: zlib wrapper test # ~7.5mn if: branch = master script: From 52dc6ff00eb653091d4ee01c8f445325a2ece09c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Nov 2019 14:39:43 -0800 Subject: [PATCH 10/10] travis: updated mingw cross compilation test - only compiles zstd CLI - enforces zero warning tolerance policy note : mingw compilation of the library complains (warning) about -fPIC flag. It mostly ignores it, so it's not a big issue. Still, it would need to be fixed if library cross-compilation becomes part of CI. At this stage, it's unclear if this is a scenario. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7d027f9..57242a48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,7 @@ matrix: - make cmakebuild - name: static analyzer scanbuild # ~26mn - dist: trusty # note : it's important to pin down a version of static analyzer, since newer versions find different false positives + dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives script: - make staticAnalyze @@ -123,7 +123,7 @@ matrix: - name: mingw cross-compilation script : - sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; - - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ make + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd - name: Valgrind + Fuzz Test Stack Mode # ~ 7mn script: