diff --git a/.gitignore b/.gitignore index c939f122..220a1e0e 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ _zstdbench/ *.swp .DS_Store googletest/ +*.d diff --git a/.travis.yml b/.travis.yml index 972c1b4b..f328f480 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: packages: - gcc-4.8 - g++-4.8 - env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" + env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" @@ -69,7 +69,7 @@ matrix: sudo: required install: - export CXX="g++-4.8" CC="gcc-4.8" - env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd pzstd32 && make -C contrib/pzstd googletest32 && make -C contrib/pzstd test32 && make -C contrib/pzstd clean" + env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" addons: apt: packages: diff --git a/appveyor.yml b/appveyor.yml index 6345c7b3..fbdc30c4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,10 +50,9 @@ build_script: ECHO *** && ECHO *** Building pzstd for %PLATFORM% && ECHO *** && - ECHO make -C contrib\pzstd pzstd && - make -C contrib\pzstd pzstd && make -C contrib\pzstd googletest-mingw64 && - make -C contrib\pzstd test && + make -C contrib\pzstd all && + make -C contrib\pzstd check && make -C contrib\pzstd clean ) - if [%COMPILER%]==[gcc] ( diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index e30be0be..b998c1ff 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -7,20 +7,69 @@ # of patent rights can be found in the PATENTS file in the same directory. # ########################################################################## +# Standard variables for installation +DESTDIR ?= +PREFIX ?= /usr/local +BINDIR := $(DESTDIR)$(PREFIX)/bin + ZSTDDIR = ../../lib PROGDIR = ../../programs -CPPFLAGS = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. -CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) +# External program to use to run tests, e.g. qemu or valgrind +TESTPROG ?= +# Flags to pass to the tests +TESTFLAGS ?= + +# We use gcc/clang to generate the header dependencies of files +DEPFLAGS = -MMD -MP -MF $*.Td +POSTCOMPILE = mv -f $*.Td $*.d + +# CFLAGS, CXXFLAGS, and LDFLAGS are for the users to override +CFLAGS ?= -O3 -Wall -Wextra +CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -std=c++11 +LDFLAGS ?= + +# Googletest default flags +PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. +GTEST_INC = -isystem googletest/googletest/include + +PZSTD_CCXXFLAGS = $(PZSTD_INC) $(GTEST_INC) +PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS) +PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) +PZSTD_LDFLAGS = +EXTRA_FLAGS = +ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS) +ALL_CXXFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS) +ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS) -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +# gtest libraries need to go before "-lpthread" because they depend on it. +GTEST_LIB = -L googletest/build/googlemock/gtest +LIBS = $(GTEST_LIB) -lpthread + +# Compilation commands +LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -o $@ +CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@ +CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@ + +# Get a list of all zstd files so we rebuild the static library when we need to +ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \ + $(wildcard $(ZSTDDIR)/common/*.h) +ZSTDCOMP_FILES := $(wildcard $(ZSTDDIR)/compress/*.c) \ + $(wildcard $(ZSTDDIR)/compress/*.h) +ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \ + $(wildcard $(ZSTDDIR)/decompress/*.h) +ZSTDPROG_FILES := $(wildcard $(PROGDIR)/*.c) \ + $(wildcard $(PROGDIR)/*.h) +ZSTD_FILES := $(wildcard $(ZSTDDIR)/*.h) \ + $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ + $(ZSTDPROG_FILES) + +# List all the pzstd source files so we can determine their dependencies +PZSTD_SRCS := $(wildcard *.cpp) +PZSTD_TESTS := $(wildcard test/*.cpp) +UTILS_TESTS := $(wildcard utils/test/*.cpp) +ALL_SRCS := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS) # Define *.exe as extension for Windows systems @@ -30,89 +79,168 @@ else EXT = endif -.PHONY: default all test clean test32 googletest googletest32 +# Standard targets +.PHONY: default +default: all -default: pzstd +.PHONY: check +check: + $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS) -all: pzstd +.PHONY: install +install: PZSTD_CPPFLAGS += -DNDEBUG +install: pzstd$(EXT) + install -d -m 755 $(BINDIR)/ + install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT) + +.PHONY: uninstall +uninstall: + $(RM) $(BINDIR)/pzstd$(EXT) + +# Targets for many different builds +.PHONY: all +all: PZSTD_CPPFLAGS += -DNDEBUG +all: pzstd$(EXT) tests roundtrip + +.PHONY: debug +debug: EXTRA_FLAGS += -g +debug: pzstd$(EXT) tests roundtrip + +.PHONY: tsan +tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC +tsan: PZSTD_LDFLAGS += -fsanitize=thread -pie +tsan: debug + +.PHONY: asan +asan: EXTRA_FLAGS += -fsanitize=address +asan: debug + +.PHONY: ubsan +ubsan: EXTRA_FLAGS += -fsanitize=undefined +ubsan: debug + +.PHONY: all32 +all32: EXTRA_FLAGS += -m32 +all32: all + +.PHONY: debug32 +debug32: EXTRA_FLAGS += -m32 +debug32: debug + +.PHONY: asan32 +asan32: EXTRA_FLAGS += -m32 +asan32: asan + +.PHONY: tsan32 +tsan32: EXTRA_FLAGS += -m32 +tsan32: tsan + +.PHONY: ubsan32 +ubsan32: EXTRA_FLAGS += -m32 +ubsan32: ubsan + +# Run long round trip tests +.PHONY: roundtripcheck +roundtripcheck: roundtrip check + $(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS) + +# Build the main binary +pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a + $(LD_COMMAND) + +# Target that depends on all the tests +.PHONY: tests +tests: EXTRA_FLAGS += -Wno-deprecated-declarations +tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS))) + +# Build the round trip tests +.PHONY: roundtrip +roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations +roundtrip: test/RoundTripTest$(EXT) + +# Use the static library that zstd builds for simplicity and +# so we get the compiler options correct +$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES) + $(MAKE) -C $(ZSTDDIR) libzstd CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" -libzstd.a: $(ZSTD_FILES) - $(MAKE) -C $(ZSTDDIR) libzstd - @cp $(ZSTDDIR)/libzstd.a . +# Rules to build the tests +test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \ + Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a + $(LD_COMMAND) -Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h - $(CXX) $(FLAGS) -c Pzstd.cpp -o $@ +test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main +test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \ + SkippableFrame.o $(ZSTDDIR)/libzstd.a + $(LD_COMMAND) -SkippableFrame.o: SkippableFrame.h SkippableFrame.cpp utils/*.h - $(CXX) $(FLAGS) -c SkippableFrame.cpp -o $@ +utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main +utils/test/%Test$(EXT): utils/test/%Test.o + $(LD_COMMAND) -Options.o: Options.h Options.cpp - $(CXX) $(FLAGS) -c Options.cpp -o $@ -main.o: main.cpp *.h utils/*.h - $(CXX) $(FLAGS) -c main.cpp -o $@ - -pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a - $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread - -libzstd32.a: $(ZSTD_FILES) - $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32" - @cp $(ZSTDDIR)/libzstd.a libzstd32.a - -Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h - $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@ - -SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h - $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@ - -Options32.o: Options.h Options.cpp - $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@ - -main32.o: main.cpp *.h utils/*.h - $(CXX) -m32 $(FLAGS) -c main.cpp -o $@ - -pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a - $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread +GTEST_CMAKEFLAGS = +# Install googletest +.PHONY: googletest +googletest: PZSTD_CCXXFLAGS += -fPIC googletest: @$(RM) -rf googletest @git clone https://github.com/google/googletest @mkdir -p googletest/build - @cd googletest/build && cmake .. && make + @cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE) -googletest32: - @$(RM) -rf googletest - @git clone https://github.com/google/googletest - @mkdir -p googletest/build - @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make - -googletest-mingw64: - $(RM) -rf googletest - git clone https://github.com/google/googletest - mkdir -p googletest/build - cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE) - -test: - $(MAKE) libzstd.a - $(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror" - $(MAKE) -C utils/test clean - $(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror" - $(MAKE) -C test clean - $(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror" - -test32: - $(MAKE) libzstd.a MOREFLAGS="-m32" - $(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" - $(MAKE) -C utils/test clean - $(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" - $(MAKE) -C test clean - $(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" +.PHONY: googletest32 +googletest32: PZSTD_CCXXFLAGS += -m32 +googletest32: googletest +.PHONY: googletest-mingw64 +googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles" +googletest-mingw64: googletest +.PHONY: clean clean: + $(RM) -f *.o pzstd$(EXT) *.Td *.d + $(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d + $(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d + $(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d $(MAKE) -C $(ZSTDDIR) clean - $(MAKE) -C utils/test clean - $(MAKE) -C test clean - @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT) @echo Cleaning completed + + +# Cancel implicit rules +%.o: %.c +%.o: %.cpp + +# Object file rules +%.o: %.c + $(CC_COMMAND) + $(POSTCOMPILE) + +$(PROGDIR)/%.o: $(PROGDIR)/%.c + $(CC_COMMAND) + $(POSTCOMPILE) + +%.o: %.cpp + $(CXX_COMMAND) + $(POSTCOMPILE) + +test/%.o: test/%.cpp + $(CXX_COMMAND) + $(POSTCOMPILE) + +utils/test/%.o: utils/test/%.cpp + $(CXX_COMMAND) + $(POSTCOMPILE) + +# Dependency file stuff +.PRECIOUS: %.d test/%.d utils/test/%.d + +# Include rules that specify header file dependencies +-include $(patsubst %,%.d,$(basename $(ALL_SRCS))) diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile deleted file mode 100644 index 4f6ba999..00000000 --- a/contrib/pzstd/test/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# ########################################################################## -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# ########################################################################## - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - -PZSTDDIR = .. -PROGDIR = ../../../programs -ZSTDDIR = ../../../lib - -# Set GTEST_INC and GTEST_LIB to work with your install of gtest -GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include -GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest -GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB) -CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. - -CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 -Wno-deprecated-declarations -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) - -datagen.o: $(PROGDIR)/datagen.* - $(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@ - -%: %.cpp *.h datagen.o - $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread - -.PHONY: test clean - -test: OptionsTest PzstdTest - @./OptionsTest$(EXT) - @./PzstdTest$(EXT) - -roundtrip: RoundTripTest - @./RoundTripTest$(EXT) - -clean: - @rm -f datagen.o OptionsTest PzstdTest RoundTripTest diff --git a/contrib/pzstd/utils/test/Makefile b/contrib/pzstd/utils/test/Makefile deleted file mode 100644 index b9ea73e3..00000000 --- a/contrib/pzstd/utils/test/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -# ########################################################################## -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# ########################################################################## - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - -PZSTDDIR = ../.. - -# Set GTEST_INC and GTEST_LIB to work with your install of gtest -GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include -GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest - -CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) - -%: %.cpp - $(CXX) $(FLAGS) $^ -o $@$(EXT) -lgtest -lgtest_main -lpthread - -.PHONY: test clean - -test: BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest - @./BufferTest$(EXT) - @./RangeTest$(EXT) - @./ScopeGuardTest$(EXT) - @./ThreadPoolTest$(EXT) - @./WorkQueueTest$(EXT) - -clean: - @rm -f BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest