From 030ac243a0f38764b9faf7e68ec0307f0e16d60c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Feb 2017 16:49:34 -0800 Subject: [PATCH 01/11] Changed Makefile to generate zstd with .gz support by default .gz support is detected by a runtime test. --- programs/.gitignore | 1 + programs/Makefile | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/programs/.gitignore b/programs/.gitignore index 24f96cf4..eeaf051d 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -8,6 +8,7 @@ zstd-decompress *.o *.ko default.profraw +have_zlib # Executables *.exe diff --git a/programs/Makefile b/programs/Makefile index 4392939d..fce47b12 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -41,7 +41,6 @@ ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o ifeq ($(ZSTD_LEGACY_SUPPORT), 0) -CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 ZSTDLEGACY_FILES:= else ZSTD_LEGACY_SUPPORT:=1 @@ -66,6 +65,12 @@ else EXT = endif +# zlib detection +HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo 1 || echo 0) +ifeq ($(HAVE_ZLIB), 1) +ZLIBCPP = -DZSTD_GZDECOMPRESS +ZLIBLD = -lz +endif .PHONY: default all clean clean_decomp_o install uninstall generate_res @@ -75,13 +80,24 @@ all: zstd $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) -zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -zstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o +zstd-internal : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +zstd-internal : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o +ifeq ($(HAVE_ZLIB), 1) + @echo "==> building zstd with .gz decompression support " +else + @echo "==> no zlib, building zstd with .zst support only (no .gz support) " +endif ifneq (,$(filter Windows%,$(OS))) windres/generate_res.bat endif - $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) + $(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS) +zstd : CPPFLAGS += $(ZLIBCPP) +zstd : LDFLAGS += $(ZLIBLD) +zstd : zstd-internal + +zstd-nogz : HAVE_ZLIB=0 +zstd-nogz : zstd-internal zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c @@ -118,17 +134,6 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) -gzstd: - @echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo found zlib || echo did not found zlib - @if [ -s have_zlib ]; then \ - echo building gzstd with .gz decompression support \ - && $(RM) have_zlib$(EXT) fileio.o \ - && CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd; \ - else \ - echo "WARNING : no zlib, building gzstd with only .zst files support : NO .gz SUPPORT !!!" \ - && $(MAKE) zstd; \ - fi - zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD zstdmt: LDFLAGS += -lpthread zstdmt: zstd @@ -141,7 +146,7 @@ clean: @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ - *.gcda default.profraw + *.gcda default.profraw have_zlib @echo Cleaning completed clean_decomp_o: From c3cba9d8580ca31cd6cb4103fd86919631c146be Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Feb 2017 17:12:50 -0800 Subject: [PATCH 02/11] fixed silent conversion warnings in GZDECOMPRESS path --- programs/fileio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 960c6e3d..5b9c91bc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -743,8 +743,8 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */ strm.next_out = ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; - strm.avail_in = ress->srcBufferLoaded; + strm.avail_out = (uInt)ress->dstBufferSize; + strm.avail_in = (uInt)ress->srcBufferLoaded; strm.next_in = (z_const unsigned char*)ress->srcBuffer; for ( ; ; ) { @@ -753,7 +753,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); if (ress->srcBufferLoaded == 0) break; strm.next_in = (z_const unsigned char*)ress->srcBuffer; - strm.avail_in = ress->srcBufferLoaded; + strm.avail_in = (uInt)ress->srcBufferLoaded; } ret = inflate(&strm, Z_NO_FLUSH); if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } @@ -762,7 +762,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); outFileSize += decompBytes; strm.next_out = ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; + strm.avail_out = (uInt)ress->dstBufferSize; } } if (ret == Z_STREAM_END) break; From c2a4632789bcf8af3e69d95d358ccb8dd99c6e1d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Feb 2017 20:54:14 -0800 Subject: [PATCH 03/11] release builds use less debug symbols and warnings release build are triggered through either `make`, or their specific target `make zstd-release` and `make lib-release`. --- Makefile | 7 ++++++- lib/Makefile | 13 ++++++++----- programs/Makefile | 17 ++++++++++------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 14d1510a..d86db7cb 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ EXT = endif .PHONY: default -default: lib zstd +default: lib zstd-release .PHONY: all all: allmost @@ -50,6 +50,11 @@ zstd: @$(MAKE) -C $(PRGDIR) $@ cp $(PRGDIR)/zstd$(EXT) . +.PHONY: zstd-release +zstd-release: + @$(MAKE) -C $(PRGDIR) + cp $(PRGDIR)/zstd$(EXT) . + .PHONY: zstdmt zstdmt: @$(MAKE) -C $(PRGDIR) $@ diff --git a/lib/Makefile b/lib/Makefile index c4a5ecb9..bef69543 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -22,10 +22,10 @@ VERSION?= $(LIBVER) CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wpointer-arith -CFLAGS += $(MOREFLAGS) +DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) @@ -59,7 +59,7 @@ LIBZSTD = libzstd.$(SHARED_EXT_VER) .PHONY: default all clean install uninstall -default: lib +default: lib-release all: lib @@ -85,6 +85,9 @@ libzstd : $(LIBZSTD) lib: libzstd.a libzstd +lib-release: DEBUGFLAGS := +lib-release: lib + clean: @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib @$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o diff --git a/programs/Makefile b/programs/Makefile index fce47b12..ce90bd45 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -26,10 +26,10 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wpointer-arith -CFLAGS += $(MOREFLAGS) +DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) @@ -74,7 +74,7 @@ endif .PHONY: default all clean clean_decomp_o install uninstall generate_res -default: zstd +default: zstd-release all: zstd @@ -92,12 +92,15 @@ ifneq (,$(filter Windows%,$(OS))) endif $(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS) +zstd-nogz : HAVE_ZLIB=0 +zstd-nogz : zstd-internal + zstd : CPPFLAGS += $(ZLIBCPP) zstd : LDFLAGS += $(ZLIBLD) zstd : zstd-internal -zstd-nogz : HAVE_ZLIB=0 -zstd-nogz : zstd-internal +zstd-release: DEBUGFLAGS := +zstd-release: zstd zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c From b02ac8d613bdaf7baecc8548b3a56e28586d70e4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Feb 2017 08:43:06 -0800 Subject: [PATCH 04/11] fixed pointer conversion warnings (C++) in gz module --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5b9c91bc..a9e1574a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -742,7 +742,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co strm.avail_in = Z_NULL; if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */ - strm.next_out = ress->dstBuffer; + strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_in = (uInt)ress->srcBufferLoaded; strm.next_in = (z_const unsigned char*)ress->srcBuffer; @@ -761,7 +761,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co if (decompBytes) { if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); outFileSize += decompBytes; - strm.next_out = ress->dstBuffer; + strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; } } From 21eb80d4850e4156d4adc3feb7d77b4f6b1c37ab Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Feb 2017 14:25:07 -0800 Subject: [PATCH 05/11] remove zlib detection artefact result of compilation test is sent to /dev/null --- NEWS | 1 + programs/Makefile | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index f404f6e3..072caee5 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.1.3 +cli : zstd can decompress .gz files. Feature can be turned off by targeting `make zstd-nogz` or setting `make HAVE_ZLIB=0` cli : new : experimental target `make zstdmt`, with multi-threading support cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski cli : fix zstdless on Mac OS-X, by Andrew Janke diff --git a/programs/Makefile b/programs/Makefile index ce90bd45..f94bffd4 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -66,7 +66,8 @@ EXT = endif # zlib detection -HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo 1 || echo 0) +VOID = /dev/null +HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o $(VOID) -x c - -lz && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz From 762ddeeb9ee6f83fdb460a96a68a7881f5dc1c4b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Feb 2017 14:35:42 -0800 Subject: [PATCH 06/11] fixed zstdmt compilation under Windows minGW/MSYS2, by @inikep --- programs/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/Makefile b/programs/Makefile index f94bffd4..9746ec4e 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -139,7 +139,9 @@ zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD +ifeq (,$(filter Windows%,$(OS))) zstdmt: LDFLAGS += -lpthread +endif zstdmt: zstd generate_res: From b4016ff02fde7b3f53172934b02821f56e1a5671 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 3 Feb 2017 16:42:07 -0800 Subject: [PATCH 07/11] Add cover dictionary training to zstd.1 Tested with `make install && man zstd` and visual inspection. --- programs/zstd.1 | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/programs/zstd.1 b/programs/zstd.1 index db79be59..384f69e3 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -251,6 +251,30 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1 .B \-s# dictionary selectivity level (default: 9) the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size. +.TP +.B \--cover=k=#,d=# + Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR. + Selects segments of size \fIk\fR with the highest score to put in the dictionary. + The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR. + Generally \fId\fR should be in the range [6, 24]. + Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048]. + Example: \fB--train --cover=k=64,d=8 FILEs\fR. +.TP +.B \--optimize-cover[=steps=#,k=#,d=#] + If \fIsteps\fR is not specified, the default value of 32 is used. + If \fIk\fR is not specified, \fIsteps\fR values in [16, 2048] are checked for each value of \fId\fR. + If \fId\fR is not specified, the values checked are [6, 8, ..., 16]. + + Runs the cover dictionary builder for each parameter set saves the optimal parameters and dictionary. + Prints the optimal parameters and writes the optimal dictionary to the output file. + Supports multithreading if \fBzstd\fR is compiled with threading support. + + The parameter \fIk\fR is more sensitve than \fId\fR, and is faster to optimize over. + Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set. + Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024]). + \fBzstd --train --optimize-cover FILEs + \fBzstd --train --optimize-cover=d=d,steps=512 FILEs +.TP .SH BENCHMARK .TP From 6f31b76d1d9af3268de2da027a0e88b6cb07e07e Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Sat, 4 Feb 2017 21:39:41 -0800 Subject: [PATCH 08/11] removed gzstd test from travis this target does no longer exist --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ba9f9965..90306dab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ matrix: # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C tests test-gzstd && make -C programs clean && make -C tests versionsTest" + - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest" os: linux sudo: required From 613087c02bd6bf1a95c71790844bf9d770e54e0e Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Sat, 4 Feb 2017 23:36:12 -0800 Subject: [PATCH 09/11] Silence zlib detection routine When it fails, $(CC) sends error message into stderr redirected to /dev/null --- programs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index 9746ec4e..599bef69 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,7 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o $(VOID) -x c - -lz && echo 1 || echo 0) +HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz From b54e235bf305f4760bcc26200857cfa7a59ba8cd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Feb 2017 10:22:58 -0800 Subject: [PATCH 10/11] fixed Mac OS-X specific directory in $(RM) list these directories are now removed with -r command --- lib/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Makefile b/lib/Makefile index bef69543..05dd2bc9 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -89,7 +89,9 @@ lib-release: DEBUGFLAGS := lib-release: lib clean: - @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib + @$(RM) -r *.dSYM # Mac OS-X specific + @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc + @$(RM) dll/libzstd.dll dll/libzstd.lib @$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o @echo Cleaning library completed From 5962e5865959ea3f2bae9b0bec7c702ae599fd97 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 5 Feb 2017 18:09:35 -0800 Subject: [PATCH 11/11] updated NEWS for v1.1.3 --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 072caee5..4f746305 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v1.1.3 -cli : zstd can decompress .gz files. Feature can be turned off by targeting `make zstd-nogz` or setting `make HAVE_ZLIB=0` +cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) cli : new : experimental target `make zstdmt`, with multi-threading support +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski cli : fix zstdless on Mac OS-X, by Andrew Janke cli : fix #232 "compress non-files"