Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

.travis.yml

@@ -33,7 +33,7 @@ matrix:
 
 
     # Standard Ubuntu 12.04 LTS Server Edition 64 bit
-    - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C tests test-gzstd && make -C programs clean && make -C tests versionsTest"
+    - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest"
       os: linux
       sudo: required
 

Makefile

@@ -23,7 +23,7 @@ EXT =
 endif
 
 .PHONY: default
-default: lib zstd
+default: lib zstd-release
 
 .PHONY: all
 all: allmost
@@ -50,6 +50,11 @@ zstd:
 	@$(MAKE) -C $(PRGDIR) $@
 	cp $(PRGDIR)/zstd$(EXT) .
 
+.PHONY: zstd-release
+zstd-release:
+	@$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd$(EXT) .
+
 .PHONY: zstdmt
 zstdmt:
 	@$(MAKE) -C $(PRGDIR) $@

NEWS

@@ -1,5 +1,7 @@
 v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
 cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell
 cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
 cli : fix zstdless on Mac OS-X, by Andrew Janke
 cli : fix #232 "compress non-files"

lib/Makefile

@@ -22,10 +22,10 @@ VERSION?= $(LIBVER)
 
 CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
 CFLAGS  ?= -O3
-CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-           -Wpointer-arith
-CFLAGS  += $(MOREFLAGS)
+DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+           -Wstrict-prototypes -Wundef -Wpointer-arith
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 
@@ -59,7 +59,7 @@ LIBZSTD = libzstd.$(SHARED_EXT_VER)
 
 .PHONY: default all clean install uninstall
 
-default: lib
+default: lib-release
 
 all: lib
 
@@ -85,8 +85,13 @@ libzstd : $(LIBZSTD)
 
 lib: libzstd.a libzstd
 
+lib-release: DEBUGFLAGS :=
+lib-release: lib
+
 clean:
-	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
+	@$(RM) -r *.dSYM   # Mac OS-X specific
+	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+	@$(RM) dll/libzstd.dll dll/libzstd.lib
 	@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
 	@echo Cleaning library completed
 

programs/.gitignore

@@ -8,6 +8,7 @@ zstd-decompress
 *.o
 *.ko
 default.profraw
+have_zlib
 
 # Executables
 *.exe

programs/Makefile

@@ -26,10 +26,10 @@ endif
 
 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder
 CFLAGS  ?= -O3
-CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-          -Wpointer-arith
-CFLAGS  += $(MOREFLAGS)
+DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+          -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+          -Wstrict-prototypes -Wundef -Wpointer-arith
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
@@ -41,7 +41,6 @@ ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
 
 ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
-CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
 ZSTDLEGACY_FILES:=
 else
 ZSTD_LEGACY_SUPPORT:=1
@@ -66,22 +65,43 @@ else
 EXT =
 endif
 
+# zlib detection
+VOID = /dev/null
+HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0)
+ifeq ($(HAVE_ZLIB), 1)
+ZLIBCPP = -DZSTD_GZDECOMPRESS
+ZLIBLD = -lz
+endif
 
 .PHONY: default all clean clean_decomp_o install uninstall generate_res
 
-default: zstd
+default: zstd-release
 
 all: zstd
 
 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 
-zstd  : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd  : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+zstd-internal : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd-internal : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+ifeq ($(HAVE_ZLIB), 1)
+	@echo "==> building zstd with .gz decompression support "
+else
+	@echo "==> no zlib, building zstd with .zst support only (no .gz support) "
+endif
 ifneq (,$(filter Windows%,$(OS)))
 	windres/generate_res.bat
 endif
-	$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
+	$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
 
+zstd-nogz : HAVE_ZLIB=0
+zstd-nogz : zstd-internal
+
+zstd : CPPFLAGS += $(ZLIBCPP)
+zstd : LDFLAGS += $(ZLIBLD)
+zstd : zstd-internal
+
+zstd-release: DEBUGFLAGS :=
+zstd-release: zstd
 
 zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
@@ -118,19 +138,10 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
 zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
 
-gzstd:
-	@echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo found zlib || echo did not found zlib
-	@if [ -s have_zlib ]; then \
-        echo building gzstd with .gz decompression support \
-        && $(RM) have_zlib$(EXT) fileio.o \
-        && CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd; \
-    else \
-        echo "WARNING : no zlib, building gzstd with only .zst files support : NO .gz SUPPORT !!!" \
-        && $(MAKE) zstd; \
-    fi
-
 zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD
+ifeq (,$(filter Windows%,$(OS)))
 zstdmt: LDFLAGS += -lpthread
+endif
 zstdmt: zstd
 
 generate_res:
@@ -141,7 +152,7 @@ clean:
 	@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
 	@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
         zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
-        *.gcda default.profraw
+        *.gcda default.profraw have_zlib
 	@echo Cleaning completed
 
 clean_decomp_o:

programs/fileio.c

@@ -742,9 +742,9 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
     strm.avail_in = Z_NULL;
     if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0;  /* see http://www.zlib.net/manual.html */
 
-    strm.next_out = ress->dstBuffer;
-    strm.avail_out = ress->dstBufferSize;
-    strm.avail_in = ress->srcBufferLoaded;
+    strm.next_out = (Bytef*)ress->dstBuffer;
+    strm.avail_out = (uInt)ress->dstBufferSize;
+    strm.avail_in = (uInt)ress->srcBufferLoaded;
     strm.next_in = (z_const unsigned char*)ress->srcBuffer;
 
     for ( ; ; ) {
@@ -753,7 +753,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
             ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
             if (ress->srcBufferLoaded == 0) break;
             strm.next_in = (z_const unsigned char*)ress->srcBuffer;
-            strm.avail_in = ress->srcBufferLoaded;
+            strm.avail_in = (uInt)ress->srcBufferLoaded;
         }
         ret = inflate(&strm, Z_NO_FLUSH);
         if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
@@ -761,8 +761,8 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
             if (decompBytes) {
                 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
                 outFileSize += decompBytes;
-                strm.next_out = ress->dstBuffer;
-                strm.avail_out = ress->dstBufferSize;
+                strm.next_out = (Bytef*)ress->dstBuffer;
+                strm.avail_out = (uInt)ress->dstBufferSize;
             }
         }
         if (ret == Z_STREAM_END) break;

programs/zstd.1

@@ -251,6 +251,30 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
 .B \-s#
  dictionary selectivity level (default: 9)
  the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.
+.TP
+.B \--cover=k=#,d=#
+ Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR.
+ Selects segments of size \fIk\fR with the highest score to put in the dictionary.
+ The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR.
+ Generally \fId\fR should be in the range [6, 24].
+ Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048].
+ Example: \fB--train --cover=k=64,d=8 FILEs\fR.
+.TP
+.B \--optimize-cover[=steps=#,k=#,d=#]
+ If \fIsteps\fR is not specified, the default value of 32 is used.
+ If \fIk\fR is not specified, \fIsteps\fR values in [16, 2048] are checked for each value of \fId\fR.
+ If \fId\fR is not specified, the values checked are [6, 8, ..., 16].
+
+ Runs the cover dictionary builder for each parameter set saves the optimal parameters and dictionary.
+ Prints the optimal parameters and writes the optimal dictionary to the output file.
+ Supports multithreading if \fBzstd\fR is compiled with threading support.
+
+ The parameter \fIk\fR is more sensitve than \fId\fR, and is faster to optimize over.
+ Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set.
+ Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024]).
+ \fBzstd --train --optimize-cover FILEs
+ \fBzstd --train --optimize-cover=d=d,steps=512 FILEs
+.TP
 
 .SH BENCHMARK
 .TP