diff --git a/Makefile b/Makefile index 1735ab86..9be0557b 100644 --- a/Makefile +++ b/Makefile @@ -31,9 +31,9 @@ VOID = /dev/null TARGET_SYSTEM ?= $(OS) ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) -EXT =.exe + EXT =.exe else -EXT = + EXT = endif ## default: Build lib-release and zstd-release @@ -65,7 +65,7 @@ lib lib-release lib-all : .PHONY: zstd zstd-release zstd zstd-release: $(Q)$(MAKE) -C $(PRGDIR) $@ - $(Q)cp $(PRGDIR)/zstd$(EXT) . + $(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT) .PHONY: zstdmt zstdmt: @@ -79,9 +79,9 @@ zlibwrapper: lib ## test: run long-duration tests .PHONY: test DEBUGLEVEL ?= 1 -test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror +test: MOREFLAGS += -g -Werror test: - MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants + DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@ diff --git a/appveyor.yml b/appveyor.yml index 91568e48..6b5b9767 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -194,7 +194,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" + SCRIPT: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" diff --git a/lib/Makefile b/lib/Makefile index 58e80e47..b8295ff0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -54,17 +54,18 @@ else CFLAGS += -O3 endif -CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed - CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= @@ -170,7 +171,29 @@ ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o) ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated vpath %.c $(ZSTD_SUBDIR) -BUILD_DIR ?= obj +UNAME := $(shell uname) +ifeq ($(UNAME), Darwin) + HASH ?= md5 +endif +ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +endif +ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH), 1) + HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | head -c 16) + HASH_DIR := conf_$(HASH_VALUE) +else + $(info warning : could not find hash function to differentiate builds with different flags) + HASH_DIR := 0 +endif + + +BUILD_DIR ?= obj/$(HASH_DIR) ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynlib ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) ZSTD_STATLIB_DIR := $(BUILD_DIR)/statlib @@ -178,7 +201,7 @@ ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) # macOS linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(shell uname), Darwin) +ifeq ($(UNAME), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) @@ -200,14 +223,20 @@ lib-all: all all: lib -libzstd.a: ARFLAGS = rcs -libzstd.a: $(ZSTD_STATLIB_OBJ) +$(ZSTD_STATLIB_DIR)/libzstd.a: ARFLAGS = rcs +$(ZSTD_STATLIB_DIR)/libzstd.a: | $(ZSTD_STATLIB_DIR) +$(ZSTD_STATLIB_DIR)/libzstd.a: $(ZSTD_STATLIB_OBJ) @echo compiling static library $(Q)$(AR) $(ARFLAGS) $@ $^ +.PHONY: libzstd.a # must be run every time +libzstd.a: $(ZSTD_STATLIB_DIR)/libzstd.a + $(Q)ln -sf $< $@ + ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) LIBZSTD = dll\libzstd.dll +$(LIBZSTD): $(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@ @@ -215,15 +244,20 @@ $(LIBZSTD): $(ZSTD_FILES) else LIBZSTD = libzstd.$(SHARED_EXT_VER) -$(LIBZSTD): CFLAGS += -fPIC -$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden -$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): CFLAGS += -fPIC +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): | $(ZSTD_DYNLIB_DIR) +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) @echo compiling dynamic library $(LIBVER) $(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links $(Q)ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) $(Q)ln -sf $@ libzstd.$(SHARED_EXT) +.PHONY: $(LIBZSTD) # must be run every time +$(LIBZSTD): $(ZSTD_DYNLIB_DIR)/$(LIBZSTD) + $(Q)ln -sf $< $@ + endif .PHONY: libzstd @@ -281,14 +315,13 @@ clean: $(Q)$(RM) -r *.dSYM # macOS-specific $(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc $(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* - $(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o - $(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/* $(ZSTD_DYNLIB_DIR)/* + $(Q)$(RM) -r obj/* @echo Cleaning library completed #----------------------------------------------------------------------------- # make install is validated only for below listed environments #----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) all: libzstd.pc @@ -324,13 +357,13 @@ $(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(PREFI endif endif -ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly)) +ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly)) PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig else PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig endif -ifneq (,$(filter $(shell uname),SunOS)) +ifneq (,$(filter $(UNAME),SunOS)) INSTALL ?= ginstall else INSTALL ?= install diff --git a/lib/README.md b/lib/README.md index ff62b668..db9170a7 100644 --- a/lib/README.md +++ b/lib/README.md @@ -168,6 +168,26 @@ file it should be linked with `dll\libzstd.dll`. For example: The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + #### Deprecated API Obsolete API on their way out are stored in directory `lib/deprecated`. diff --git a/programs/Makefile b/programs/Makefile index 8f14292f..70fb2c32 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -36,19 +36,20 @@ ZSTD_VERSION = $(LIBVER) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -GREP_OPTIONS += --color=never + GREP_OPTIONS += --color=never endif GREP = grep $(GREP_OPTIONS) ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1) -ALIGN_LOOP = -falign-loops=32 + ALIGN_LOOP = -falign-loops=32 else -ALIGN_LOOP = + ALIGN_LOOP = endif -CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -65,11 +66,7 @@ ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress ZDICT_DIR := $(ZSTDDIR)/dictBuilder ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy -vpath %.c $(ZSTDLIB_COMMON) -vpath %.c $(ZSTDLIB_COMPRESS) -vpath %.c $(ZSTDLIB_DECOMPRESS) -vpath %.c $(ZDICT_DIR) -vpath %.c $(ZSTDLEGACY_DIR) +vpath %.c $(ZSTDLIB_COMMON) $(ZSTDLIB_COMPRESS) $(ZSTDLIB_DECOMPRESS) $(ZDICT_DIR) $(ZSTDLEGACY_DIR) ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) @@ -81,7 +78,7 @@ ZSTD_LEGACY_SUPPORT ?= 5 ZSTDLEGACY_SRC := ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif endif @@ -94,19 +91,44 @@ ZSTD_CLI_SRC := $(wildcard *.c) ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o) ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) +ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o) + +UNAME := $(shell uname) +ifeq ($(UNAME), Darwin) + HASH ?= md5 +endif +ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +endif +ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH), 1) + HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) | $(HASH) | head -c 16) + HASH_DIR := conf_$(HASH_VALUE) +else + $(info warning : could not find hash function to differentiate builds with different flags) + HASH_DIR := 0 +endif + +BUILD_DIR ?= obj/$(HASH_DIR) +ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ)) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -RES64_FILE = windres/zstd64.res -RES32_FILE = windres/zstd32.res + EXT =.exe + RES64_FILE = windres/zstd64.res + RES32_FILE = windres/zstd32.res ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine))) RES_FILE = $(RES64_FILE) else RES_FILE = $(RES32_FILE) endif else -EXT = + EXT = endif VOID = /dev/null @@ -119,54 +141,54 @@ NO_THREAD_MSG := ==> no threads, building without multithreading support HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c) HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0) ifeq ($(HAVE_THREAD), 1) -THREAD_MSG := ==> building with threading support -THREAD_CPP := -DZSTD_MULTITHREAD -THREAD_LD := -pthread + THREAD_MSG := ==> building with threading support + THREAD_CPP := -DZSTD_MULTITHREAD + THREAD_LD := -pthread else -THREAD_MSG := $(NO_THREAD_MSG) + THREAD_MSG := $(NO_THREAD_MSG) endif # zlib detection NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c) ifeq ($(HAVE_ZLIB), 1) -ZLIB_MSG := ==> building zstd with .gz compression support -ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS -ZLIBLD = -lz + ZLIB_MSG := ==> building zstd with .gz compression support + ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS + ZLIBLD = -lz else -ZLIB_MSG := $(NO_ZLIB_MSG) + ZLIB_MSG := $(NO_ZLIB_MSG) endif # lzma detection NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c) ifeq ($(HAVE_LZMA), 1) -LZMA_MSG := ==> building zstd with .xz/.lzma compression support -LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS -LZMALD = -llzma + LZMA_MSG := ==> building zstd with .xz/.lzma compression support + LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS + LZMALD = -llzma else -LZMA_MSG := $(NO_LZMA_MSG) + LZMA_MSG := $(NO_LZMA_MSG) endif # lz4 detection NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include \n$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c) ifeq ($(HAVE_LZ4), 1) -LZ4_MSG := ==> building zstd with .lz4 compression support -LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS -LZ4LD = -llz4 + LZ4_MSG := ==> building zstd with .lz4 compression support + LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS + LZ4LD = -llz4 else -LZ4_MSG := $(NO_LZ4_MSG) + LZ4_MSG := $(NO_LZ4_MSG) endif # explicit backtrace enable/disable for Linux & Darwin ifeq ($(BACKTRACE), 0) -DEBUGFLAGS += -DBACKTRACE_ENABLE=0 + DEBUGFLAGS += -DBACKTRACE_ENABLE=0 endif ifeq (,$(filter Windows%, $(OS))) ifeq ($(BACKTRACE), 1) -DEBUGFLAGS += -DBACKTRACE_ENABLE=1 -DEBUGFLAGS_LD += -rdynamic + DEBUGFLAGS += -DBACKTRACE_ENABLE=1 + DEBUGFLAGS_LD += -rdynamic endif endif @@ -180,18 +202,23 @@ all: zstd .PHONY: allVariants allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder +$(BUILD_DIR)/zstd : $(ZSTD_OBJ) + @echo "$(THREAD_MSG)" + @echo "$(ZLIB_MSG)" + @echo "$(LZMA_MSG)" + @echo "$(LZ4_MSG)" + $(Q)$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) + +.PHONY: zstd zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) endif -zstd : $(ZSTDLIB_LOCAL_OBJ) $(ZSTD_CLI_OBJ) - @echo "$(THREAD_MSG)" - @echo "$(ZLIB_MSG)" - @echo "$(LZMA_MSG)" - @echo "$(LZ4_MSG)" - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) +zstd : $(BUILD_DIR)/zstd + $(Q)ln -sf $< $@ + @echo zstd build completed .PHONY: zstd-release zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0 @@ -292,6 +319,7 @@ clean: zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \ zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT) + $(Q)$(RM) -r obj/* @echo Cleaning completed MD2ROFF = ronn @@ -324,34 +352,32 @@ preview-man: clean-man man # Generate .h dependencies automatically -MKDIR = mkdir +DEPFLAGS = -MT $@ -MMD -MP -MF -DEPDIR := .deps -DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d +$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR) + @echo $@ + $(Q)$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $< -COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c +MKDIR ?= mkdir +$(BUILD_DIR): ; $(Q)$(MKDIR) -p $@ -%.o : %.c -%.o : %.c $(DEPDIR)/%.d | $(DEPDIR) - $(COMPILE.c) $(OUTPUT_OPTION) $< - -$(DEPDIR): ; $(Q)$(MKDIR) -p $@ - -DEPFILES := $(ZSTD_ALL_SRC:%.c=$(DEPDIR)/%.d) +DEPFILES := $(ZSTD_OBJ:.o=.d) $(DEPFILES): include $(wildcard $(DEPFILES)) + #----------------------------------------------------------------------------- # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) +UNAME := $(shell uname) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) EGREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -EGREP_OPTIONS += --color=never + EGREP_OPTIONS += --color=never endif EGREP = egrep $(EGREP_OPTIONS) AWK = awk @@ -389,17 +415,17 @@ datarootdir ?= $(PREFIX)/share mandir ?= $(datarootdir)/man man1dir ?= $(mandir)/man1 -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -MANDIR ?= $(PREFIX)/man -MAN1DIR ?= $(MANDIR)/man1 +ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS)) + MANDIR ?= $(PREFIX)/man + MAN1DIR ?= $(MANDIR)/man1 else -MAN1DIR ?= $(man1dir) + MAN1DIR ?= $(man1dir) endif -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall +ifneq (,$(filter $(UNAME),SunOS)) + INSTALL ?= ginstall else -INSTALL ?= install + INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) diff --git a/programs/README.md b/programs/README.md index 53f716bd..cf7f5ba4 100644 --- a/programs/README.md +++ b/programs/README.md @@ -3,7 +3,7 @@ Command Line Interface for Zstandard library Command Line Interface (CLI) can be created using the `make` command without any additional parameters. There are however other Makefile targets that create different variations of CLI: -- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd formats +- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and supports decompression of legacy zstd formats - `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats - `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats - `zstd-compress` : version of CLI which can only compress into zstd format @@ -147,76 +147,91 @@ FILE : a filename Arguments : -# : # compression level (1-19, default: 3) -d : decompression - -D file: use `file` as Dictionary - -o file: result stored into `file` (only if 1 input file) - -f : overwrite output without prompting and (de)compress links + -D DICT: use DICT as Dictionary for compression or decompression + -o file: result stored into `file` (only 1 output file) + -f : overwrite output without prompting, also (de)compress links --rm : remove source file(s) after successful de/compression -k : preserve source file(s) (default) -h/-H : display help/long help and exit Advanced arguments : -V : display Version number and exit + -c : force write to standard output, even if it is the console -v : verbose mode; specify multiple times to increase verbosity -q : suppress warnings; specify twice to suppress errors too - -c : force write to standard output, even if it is the console - -l : print information about zstd compressed files ---exclude-compressed: only compress files that are not previously compressed +--no-progress : do not display the progress counter + -r : operate recursively on directories +--filelist FILE : read list of files to operate upon from FILE +--output-dir-flat DIR : processed files are stored into DIR +--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure +--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled). If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate). +-- : All arguments after "--" are treated as files + +Advanced compression arguments : --ultra : enable levels beyond 19, up to 22 (requires more memory) --long[=#]: enable long distance matching with given window log (default: 27) --fast[=#]: switch to very fast compression levels (default: 1) --adapt : dynamically adapt compression level to I/O conditions ---stream-size=# : optimize compression parameters for streaming input of given number of bytes ---size-hint=# optimize compression parameters for streaming input of approximately this size ---target-compressed-block-size=# : make compressed block near targeted size -T# : spawns # compression threads (default: 1, 0==# cores) -B# : select size of each job (default: 0==automatic) +--single-thread : use a single thread for both I/O and compression (result slightly different than -T1) --rsyncable : compress using a rsync-friendly method (-B sets block size) ---no-dictID : don't write dictID into header (dictionary compression) ---[no-]check : integrity check (default: enabled) +--exclude-compressed: only compress files that are not already compressed +--stream-size=# : specify size of streaming input from `stdin` +--size-hint=# optimize compression parameters for streaming input of approximately this size +--target-compressed-block-size=# : generate compressed block of approximately targeted size +--no-dictID : don't write dictID into header (dictionary compression only) --[no-]compress-literals : force (un)compressed literals - -r : operate recursively on directories ---output-dir-flat[=directory]: all resulting files stored into `directory`. --format=zstd : compress files to the .zst format (default) --format=gzip : compress files to the .gz format +--format=xz : compress files to the .xz format +--format=lzma : compress files to the .lzma format +--format=lz4 : compress files to the .lz4 format + +Advanced decompression arguments : + -l : print information about zstd compressed files --test : test compressed file integrity ---[no-]sparse : sparse mode (default: disabled) -M# : Set a memory usage limit for decompression ---no-progress : do not display the progress bar --- : All arguments after "--" are treated as files +--[no-]sparse : sparse mode (default: disabled) Dictionary builder : --train ## : create a dictionary from a training set of files --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) - -o file : `file` is dictionary name (default: dictionary) + -o DICT : DICT is dictionary name (default: dictionary) --maxdict=# : limit dictionary to specified size (default: 112640) --dictID=# : force dictionary ID to specified value (default: random) Benchmark arguments : -b# : benchmark file(s), using # compression level (default: 3) - -e# : test all compression levels from -bX to # (default: 1) + -e# : test all compression levels successively from -b# to -e# (default: 1) -i# : minimum evaluation time in seconds (default: 3s) -B# : cut file into independent blocks of size # (default: no block) + -S : output one benchmark result per input file (default: consolidated result) --priority=rt : set process priority to real-time ``` ### Passing parameters through Environment Variables +There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner. +Using environment variables for this purpose has security implications. +Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`. + `ZSTD_CLEVEL` can be used to modify the default compression level of `zstd` (usually set to `3`) to another value between 1 and 19 (the "normal" range). -`ZSTD_NBTHREADS` can be used to specify number of threads that `zstd` will use during compression, which by default is `1`. + +`ZSTD_NBTHREADS` can be used to specify a number of threads +that `zstd` will use for compression, which by default is `1`. This functionality only exists when `zstd` is compiled with multithread support. +`0` means "use as many threads as detected cpu cores on local system". The max # of threads is capped at: `ZSTDMT_NBWORKERS_MAX==200`. This functionality can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments. One such scenario is `tar --zstd`. As `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` only replace the default compression level -and number of threads, respectively, they can both be overridden by corresponding command line arguments: +and number of threads respectively, they can both be overridden by corresponding command line arguments: `-#` for compression level and `-T#` for number of threads. -There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner. -Using environment variables for this purpose has security implications. -Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`. ### Long distance matching mode The long distance matching mode, enabled with `--long`, is designed to improve diff --git a/tests/Makefile b/tests/Makefile index d347a948..c39c5ca0 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -94,23 +94,23 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus # note : broken : requires symbols unavailable from dynamic library dll: fuzzer-dll zstreamtest-dll -PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update +PHONY: zstd zstd32 zstd-nolegacy # phony: only external makefile knows how to build or update them zstd zstd32 zstd-nolegacy: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" + $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) gzstd: - $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" + $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) .PHONY: libzstd libzstd : - $(MAKE) -C $(ZSTDDIR) libzstd + $(MAKE) -C $(ZSTDDIR) libzstd DEBUGLEVEL=$(DEBUGLEVEL) %-dll : libzstd %-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd .PHONY: zstd-staticLib zstd-staticLib : - $(MAKE) -C $(ZSTDDIR) libzstd.a + $(MAKE) -C $(ZSTDDIR) libzstd.a DEBUGLEVEL=$(DEBUGLEVEL) zstdm_%.o : $(ZSTDDIR)/common/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ @@ -255,7 +255,8 @@ clean: #---------------------------------------------------------------------------------- # valgrind tests are validated only for some posix platforms #---------------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) +UNAME := $(shell uname) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) HOST_OS = POSIX valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 @@ -275,8 +276,8 @@ valgrindTest: zstd datagen fuzzer fullbench endif -ifneq (,$(filter MINGW% MSYS%,$(shell uname))) -HOST_OS = MSYS +ifneq (,$(filter MINGW% MSYS%,$(UNAME))) + HOST_OS = MSYS endif @@ -286,8 +287,8 @@ endif ifneq (,$(filter $(HOST_OS),MSYS POSIX)) DIFF:=diff -ifneq (,$(filter $(shell uname),SunOS)) -DIFF:=gdiff +ifneq (,$(filter $(UNAME),SunOS)) + DIFF:=gdiff endif .PHONY: list diff --git a/tests/libzstd_partial_builds.sh b/tests/libzstd_partial_builds.sh index b1c1e3b1..bee2dbda 100755 --- a/tests/libzstd_partial_builds.sh +++ b/tests/libzstd_partial_builds.sh @@ -22,7 +22,6 @@ mustBeAbsent() { } # default compilation : all features enabled -make clean > /dev/null $ECHO "testing default library compilation" CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog