Merge pull request #2367 from facebook/progressive_build

faster rebuild of zstd
This commit is contained in:
Yann Collet 2020-10-21 15:43:14 -07:00 committed by GitHub
commit 2224ec33ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 96 additions and 61 deletions

View File

@ -77,13 +77,13 @@ matrix:
- name: Minimal Decompressor Macros # ~5mn
script:
- make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
- make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
- make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
- make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
- make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
- make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
- make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
- make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
- make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
- make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
- make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
- make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
- name: cmake build and test check # ~6mn
script:

View File

@ -285,5 +285,6 @@
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
- if [%HOST%]==[mingw] (
set "CC=%COMPILER%" &&
make clean &&
make check
)

View File

@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
# elif (ZDICT_GCC_VERSION >= 301)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))

View File

@ -15,7 +15,10 @@
# zstd-decompress : decompressor-only version of zstd
# ##########################################################################
ZSTDDIR = ../lib
# verbose mode can be triggered by V=1 or VERBOSE=1
Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
ZSTDDIR := ../lib
# Version numbers
LIBVER_SRC := $(ZSTDDIR)/zstd.h
@ -43,41 +46,54 @@ else
ALIGN_LOOP =
endif
CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
ifeq ($(OS),Windows_NT) # MinGW assumed
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
endif
CFLAGS ?= -O3
CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
ZSTDLIB_COMMON := $(ZSTDDIR)/common
ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress
ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress
ZDICT_DIR := $(ZSTDDIR)/dictBuilder
ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
vpath %.c $(ZSTDLIB_COMMON)
vpath %.c $(ZSTDLIB_COMPRESS)
vpath %.c $(ZSTDLIB_DECOMPRESS)
vpath %.c $(ZDICT_DIR)
vpath %.c $(ZSTDLEGACY_DIR)
ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c)
ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c)
ZSTDLIB_DECOMPRESS_C := $(wildcard $(ZSTDLIB_DECOMPRESS)/*.c)
ZSTDLIB_CORE_SRC := $(ZSTDLIB_DECOMPRESS_C) $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C)
ZDICT_SRC := $(wildcard $(ZDICT_DIR)/*.c)
ZSTD_LEGACY_SUPPORT ?= 5
ZSTDLEGACY_FILES :=
ZSTDLEGACY_SRC :=
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
endif
else
endif
# Sort files in alphabetical order for reproducible builds
ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES)))
ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
ZSTDLIB_LOCAL_SRC := $(notdir $(ZSTDLIB_FULL_SRC))
ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o)
ZSTD_CLI_FILES := $(wildcard *.c)
ZSTD_CLI_OBJ := $(patsubst %.c,%.o,$(ZSTD_CLI_FILES))
ZSTD_CLI_SRC := $(wildcard *.c)
ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@ -164,15 +180,13 @@ all: zstd
.PHONY: allVariants
allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
ifneq (,$(filter Windows%,$(OS)))
zstd : $(RES_FILE)
endif
zstd : $(ZSTDLIB_FILES) $(ZSTD_CLI_OBJ)
zstd : $(ZSTDLIB_LOCAL_OBJ) $(ZSTD_CLI_OBJ)
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@ -190,12 +204,12 @@ zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
ifneq (,$(filter Windows%,$(OS)))
zstd32 : $(RES32_FILE)
endif
zstd32 : $(ZSTDLIB_FILES) $(ZSTD_CLI_FILES)
zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed
zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) $(ZSTD_CLI_OBJ)
zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd-nomt : THREAD_CPP :=
@ -222,7 +236,7 @@ zstd-noxz : zstd
# It's unclear at this stage if this is a scenario that must be supported
.PHONY: zstd-dll
zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
zstd-dll : ZSTDLIB_FILES =
zstd-dll : ZSTDLIB_FULL_SRC =
zstd-dll : $(ZSTD_CLI_OBJ)
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
@ -237,24 +251,24 @@ zstd-pgo :
./zstd -b $(PROFILE_WITH)
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
$(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o
$(RM) zstd *.o
case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
$(MAKE) zstd MOREFLAGS=-fprofile-use
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn.c fileio.c
zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c
zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS
zstd-dictBuilder: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZDICT_FILES) zstdcli.c util.c timefn.c fileio.c dibio.c
zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstdmt: zstd
@ -274,9 +288,7 @@ endif
.PHONY: clean
clean:
$(MAKE) -C $(ZSTDDIR) clean
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
$(Q)$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT)
@ -309,6 +321,28 @@ preview-man: clean-man man
man ./zstdgrep.1
man ./zstdless.1
# Generate .h dependencies automatically
MKDIR = mkdir
DEPDIR := .deps
DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d
COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c
%.o : %.c
%.o : %.c $(DEPDIR)/%.d | $(DEPDIR)
$(COMPILE.c) $(OUTPUT_OPTION) $<
$(DEPDIR): ; $(Q)$(MKDIR) -p $@
DEPFILES := $(ZSTD_ALL_SRC:%.c=$(DEPDIR)/%.d)
$(DEPFILES):
include $(wildcard $(DEPFILES))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
@ -328,7 +362,7 @@ AWK = awk
## list: Print all targets and their descriptions (if provided)
.PHONY: list
list:
@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
| $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
{ \
@ -376,34 +410,34 @@ INSTALL_MAN ?= $(INSTALL_DATA)
.PHONY: install
install: zstd
@echo Installing binaries
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/
@$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/
$(Q)$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
$(Q)ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
$(Q)ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
$(Q)ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
$(Q)$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
$(Q)$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
@echo Installing man pages
@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
@$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
@$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
$(Q)$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
$(Q)ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
$(Q)ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
$(Q)$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
$(Q)$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
@echo zstd installation completed
.PHONY: uninstall
uninstall:
@$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
@$(RM) $(DESTDIR)$(BINDIR)/zstdmt
@$(RM) $(DESTDIR)$(BINDIR)/zstd
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
@$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
@$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
$(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
$(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdless
$(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdcat
$(Q)$(RM) $(DESTDIR)$(BINDIR)/unzstd
$(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdmt
$(Q)$(RM) $(DESTDIR)$(BINDIR)/zstd
$(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
$(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
$(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
$(Q)$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
$(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
@echo zstd programs successfully uninstalled
endif