Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

NEWS

@@ -7,6 +7,9 @@ cli : fixed : status displays total amount decoded, even for file consisting of
 cli : fixed : zstdcat
 API : changed : zbuff prototypes now generate deprecation warnings
 API : changed : streaming decompression implicit reset on starting new frame
+API : added experimental : dictID retrieval functions
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+Changed : zbuff source files moved to lib/deprecated
 Changed : reduced stack memory use
 
 v1.1.1

build/VS2005/fullbench/fullbench.vcproj

@@ -363,14 +363,6 @@
 				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zbuff_compress.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
@@ -425,10 +417,6 @@
 				RelativePath="..\..\..\lib\common\mem.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\common\zbuff_static.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\zstd.h"
 				>

build/VS2005/fuzzer/fuzzer.vcproj

@@ -429,10 +429,6 @@
 				RelativePath="..\..\..\lib\common\xxhash.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\common\zbuff_static.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>

build/VS2005/zstd/zstd.vcproj

@@ -379,14 +379,6 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zbuff_compress.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
 				>
@@ -481,14 +473,6 @@
 				RelativePath="..\..\..\lib\common\xxhash.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\common\zbuff.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\common\zbuff_static.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>

build/VS2005/zstdlib/zstdlib.vcproj

@@ -360,11 +360,11 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\compress\zbuff_compress.c"
+				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
+				RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
 				>
 			</File>
 			<File
@@ -461,10 +461,6 @@
 				RelativePath="..\..\..\lib\common\zbuff.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\common\zbuff_static.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>

build/cmake/lib/CMakeLists.txt

@@ -64,25 +64,25 @@ SET(Sources
         ${LIBRARY_DIR}/common/fse_decompress.c
         ${LIBRARY_DIR}/compress/fse_compress.c
         ${LIBRARY_DIR}/compress/huf_compress.c
-        ${LIBRARY_DIR}/compress/zbuff_compress.c
         ${LIBRARY_DIR}/compress/zstd_compress.c
         ${LIBRARY_DIR}/decompress/huf_decompress.c
-        ${LIBRARY_DIR}/decompress/zbuff_decompress.c
         ${LIBRARY_DIR}/decompress/zstd_decompress.c
         ${LIBRARY_DIR}/dictBuilder/divsufsort.c
-        ${LIBRARY_DIR}/dictBuilder/zdict.c)
+        ${LIBRARY_DIR}/dictBuilder/zdict.c
+        ${LIBRARY_DIR}/deprecated/zbuff_compress.c
+        ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
 
 SET(Headers
+        ${LIBRARY_DIR}/zstd.h
         ${LIBRARY_DIR}/common/bitstream.h
         ${LIBRARY_DIR}/common/error_private.h
         ${LIBRARY_DIR}/common/zstd_errors.h
         ${LIBRARY_DIR}/common/fse.h
         ${LIBRARY_DIR}/common/huf.h
         ${LIBRARY_DIR}/common/mem.h
-        ${LIBRARY_DIR}/common/zbuff.h
         ${LIBRARY_DIR}/common/zstd_internal.h
-        ${LIBRARY_DIR}/zstd.h
+        ${LIBRARY_DIR}/dictBuilder/zdict.h
-        ${LIBRARY_DIR}/dictBuilder/zdict.h)
+        ${LIBRARY_DIR}/deprecated/zbuff.h)
 
 IF (ZSTD_LEGACY_SUPPORT)
     SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@@ -165,7 +165,7 @@ IF (UNIX)
     SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
 
     # install target
-    INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
+    INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
     INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
     INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
 

build/cmake/tests/CMakeLists.txt

@@ -50,9 +50,6 @@ ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
 TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
 
 IF (UNIX)
-    ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/zbufftest.c)
-    TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
-
     ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
     TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
 

lib/Makefile

@@ -23,24 +23,22 @@ PREFIX ?= /usr/local
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR=$(PREFIX)/include
 
-CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_
+CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
-CFLAGS ?= -O3
+CFLAGS  ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
+           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-          -Wpointer-arith
+           -Wpointer-arith
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)
+CFLAGS  += $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 
-ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c)
+ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
-ZSTD_EXCLUDE := compress/zbuff_compress.c decompress/zbuff_decompress.c
-ZSTD_FILES := $(filter-out $(ZSTD_EXCLUDE), $(ZSTD_FILES))
-
 
 ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
 else
-ZSTD_FILES+= legacy/*.c
 CPPFLAGS  += -I./legacy -DZSTD_LEGACY_SUPPORT=1
+ZSTD_FILES+= $(wildcard legacy/*.c)
 endif
 
 # OS X linker doesn't support -soname, and use different extension
@@ -90,8 +88,8 @@ libzstd : $(LIBZSTD)
 lib: libzstd.a libzstd
 
 clean:
-	@$(RM) -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
+	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
-	@$(RM) -f decompress/*.o
+	@$(RM) decompress/*.o
 	@echo Cleaning library completed
 
 #------------------------------------------------------------------------
@@ -116,7 +114,7 @@ install: libzstd.a libzstd libzstd.pc
 	@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
 	@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
 	@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
-	@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
+	@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h   # prototypes generate deprecation warnings
 	@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
 	@echo zstd static and shared library installed
 

lib/README.md

@@ -62,9 +62,9 @@ The compiled executable will require ZSTD DLL which is available at `dll\libzstd
 #### Obsolete streaming API
 
 Streaming is now provided within `zstd.h`.
-Older streaming API is still available within `common/zbuff.h`.
+Older streaming API is still available within `deprecated/zbuff.h`.
-It is now deprecated, and will be removed in a future version.
+It will be removed in a future version.
-Consider migrating towards newer streaming API in `zstd.h`.
+Consider migrating code towards newer streaming API in `zstd.h`.
 
 
 #### Miscellaneous

lib/common/zstd_common.c

@@ -16,7 +16,6 @@
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"           /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
-#include "zbuff.h"          /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
 
 
 /*-****************************************
@@ -44,16 +43,11 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 *   provides error code string from enum */
 const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
 
-
+/* ---   ZBUFF Error Management  (deprecated)   --- */
-/* **************************************************************
-*  ZBUFF Error Management
-****************************************************************/
 unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-
 const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
 
 
-
 /*=**************************************************************
 *  Custom allocator
 ****************************************************************/

lib/decompress/zstd_decompress.c

@@ -1778,6 +1778,45 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
     return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
 }
 
+/*! ZSTD_getDictID_fromDict() :
+ *  Provides the dictID stored within dictionary.
+ *  if @return == 0, the dictionary is not conformant with Zstandard specification.
+ *  It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
+{
+    if (dictSize < 8) return 0;
+    if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
+    return MEM_readLE32((const char*)dict + 4);
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ *  Provides the dictID of the dictionary loaded into `ddict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
+{
+    if (ddict==NULL) return 0;
+    return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ *  Provides the dictID required to decompressed the frame stored within `src`.
+ *  If @return == 0, the dictID could not be decoded.
+ *  This could for one of the following reasons :
+ *  - The frame does not require a dictionary to be decoded (most common case).
+ *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ *    Note : this use case also happens when using a non-conformant dictionary.
+ *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ *  - This is not a Zstandard frame.
+ *  When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
+{
+    ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 };
+    size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
+    if (ZSTD_isError(hError)) return 0;
+    return zfp.dictID;
+}
+
 
 /*! ZSTD_decompress_usingDDict() :
 *   Decompression using a pre-digested Dictionary

lib/deprecated/zbuff.h

@@ -202,6 +202,7 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
                                                const void* dict, size_t dictSize,
                                                ZSTD_parameters params, unsigned long long pledgedSrcSize);
 
+
 #endif /* ZBUFF_STATIC_LINKING_ONLY */
 
 

lib/zstd.h

@@ -53,8 +53,6 @@ extern "C" {
 *********************************************************************************************************/
 
 /*------   Version   ------*/
-ZSTDLIB_API unsigned ZSTD_versionNumber (void);  /**< returns version number of ZSTD */
-
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    1
 #define ZSTD_VERSION_RELEASE  2
@@ -65,6 +63,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void);  /**< returns version number of
 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
 
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+ZSTDLIB_API unsigned ZSTD_versionNumber (void);
 
 
 /***************************************
@@ -462,6 +461,30 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
  *  Gives the amount of memory used by a given ZSTD_DDict */
 ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
+/*! ZSTD_getDictID_fromDict() :
+ *  Provides the dictID stored within dictionary.
+ *  if @return == 0, the dictionary is not conformant with Zstandard specification.
+ *  It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromDDict() :
+ *  Provides the dictID of the dictionary loaded into `ddict`.
+ *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() :
+ *  Provides the dictID required to decompressed the frame stored within `src`.
+ *  If @return == 0, the dictID could not be decoded.
+ *  This could for one of the following reasons :
+ *  - The frame does not require a dictionary to be decoded (most common case).
+ *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ *    Note : this use case also happens when using a non-conformant dictionary.
+ *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ *  - This is not a Zstandard frame.
+ *  When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
 
 /********************************************************************
 *  Advanced streaming functions

programs/fileio.c

@@ -35,6 +35,9 @@
 #include "zstd.h"
 #ifdef ZSTD_GZDECOMPRESS
 #include "zlib.h"
+#if !defined(z_const)
+    #define z_const
+#endif
 #endif
 
 
@@ -465,6 +468,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
 ****************************************************************************/
 typedef struct {
     void*  srcBuffer;
+    size_t srcBufferLoaded;
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
@@ -582,63 +586,18 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
     }   }
 }
 
-/** FIO_decompressFrame() :
-    @return : size of decoded frame
-*/
-unsigned long long FIO_decompressFrame(dRess_t ress,
-                                       FILE* finput, size_t alreadyLoaded,
-                                       U64 alreadyDecoded)
-{
-    U64 frameSize = 0;
-    size_t readSize;
-    U32 storedSkips = 0;
-
-    ZSTD_resetDStream(ress.dctx);
-
-    /* Header loading (optional, saves one loop) */
-    {   size_t const toLoad = 9 - alreadyLoaded;   /* assumption : 9 >= alreadyLoaded */
-        size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
-        readSize = alreadyLoaded + loadedSize;
-    }
-
-    /* Main decompression Loop */
-    while (1) {
-        ZSTD_inBuffer  inBuff = { ress.srcBuffer, readSize, 0 };
-        ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
-        size_t const readSizeHint = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
-        if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
-
-        /* Write block */
-        storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips);
-        frameSize += outBuff.pos;
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)((alreadyDecoded+frameSize)>>20) );
-
-        if (readSizeHint == 0) break;   /* end of frame */
-        if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
-
-        /* Fill input buffer */
-        {   size_t const toRead = MIN(readSizeHint, ress.srcBufferSize);  /* support large skippable frames */
-            readSize = fread(ress.srcBuffer, 1, toRead, finput);
-            if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
-    }   }
-
-    FIO_fwriteSparseEnd(ress.dstFile, storedSkips);
-
-    return frameSize;
-}
-
 
 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
     @return : 0 (no error) */
-static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
+static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
 {
     size_t const blockSize = MIN(64 KB, bufferSize);
     size_t readFromInput = 1;
     unsigned storedSkips = 0;
 
-    /* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */
+    /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
-    { size_t const sizeCheck = fwrite(buffer, 1, 4, foutput);
+    { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
-      if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); }
+      if (sizeCheck != alreadyLoaded) EXM_THROW(50, "Pass-through write error"); }
 
     while (readFromInput) {
         readFromInput = fread(buffer, 1, blockSize, finput);
@@ -649,8 +608,60 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
     return 0;
 }
 
+
+/** FIO_decompressFrame() :
+    @return : size of decoded frame
+*/
+unsigned long long FIO_decompressFrame(dRess_t* ress,
+                                       FILE* finput,
+                                       U64 alreadyDecoded)
+{
+    U64 frameSize = 0;
+    U32 storedSkips = 0;
+
+    ZSTD_resetDStream(ress->dctx);
+
+    /* Header loading (optional, saves one loop) */
+    {   size_t const toRead = 9;
+        if (ress->srcBufferLoaded < toRead)
+            ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
+    }
+
+    /* Main decompression Loop */
+    while (1) {
+        ZSTD_inBuffer  inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
+        ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
+        size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
+        if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
+
+        /* Write block */
+        storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);       
+        frameSize += outBuff.pos;
+        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)((alreadyDecoded+frameSize)>>20) );
+
+        if (inBuff.pos > 0) {
+            memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
+            ress->srcBufferLoaded -= inBuff.pos;
+        }
+
+        if (readSizeHint == 0) break;   /* end of frame */
+        if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
+
+        /* Fill input buffer */
+        {   size_t const toRead = MIN(readSizeHint, ress->srcBufferSize);  /* support large skippable frames */
+            if (ress->srcBufferLoaded < toRead)
+                ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
+            if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
+    }   }
+
+    FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
+
+    return frameSize;
+}
+
+
 #ifdef ZSTD_GZDECOMPRESS
-static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded)
+static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName)
 {
     unsigned long long outFileSize = 0;
     z_stream strm;
@@ -662,32 +673,34 @@ static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, con
     strm.avail_in = Z_NULL;
     if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0;  /* see http://www.zlib.net/manual.html */
 
-    strm.next_out = ress.dstBuffer;
+    strm.next_out = ress->dstBuffer;
-    strm.avail_out = ress.dstBufferSize;
+    strm.avail_out = ress->dstBufferSize;
+    strm.avail_in = ress->srcBufferLoaded;
+    strm.next_in = (z_const unsigned char*)ress->srcBuffer;
 
     for ( ; ; ) {
-        if (alreadyLoaded) {
+        int ret;
-            strm.avail_in = alreadyLoaded;
+        if (strm.avail_in == 0) {
-            strm.next_in = (z_const unsigned char*)ress.srcBuffer;
+            ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
-            alreadyLoaded = 0;
+            if (ress->srcBufferLoaded == 0) break;
-        } else {
+            strm.next_in = (z_const unsigned char*)ress->srcBuffer;
-            if (fread(ress.srcBuffer, 1, 1/*ress.srcBufferSize*/, srcFile) == 0) break;
+            strm.avail_in = ress->srcBufferLoaded;
-            strm.next_in = (z_const unsigned char*)ress.srcBuffer;
-            strm.avail_in = 1;
         }
-        {   int const ret = inflate(&strm, Z_NO_FLUSH);
+        ret = inflate(&strm, Z_NO_FLUSH);
-            if (ret == Z_STREAM_END) break;
+        if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
-            if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
+        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
-        }
-
-        {   size_t const decompBytes = ress.dstBufferSize - strm.avail_out;
             if (decompBytes) {
-                if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
                 outFileSize += decompBytes;
-                strm.next_out = ress.dstBuffer;
+                strm.next_out = ress->dstBuffer;
-                strm.avail_out = ress.dstBufferSize;
+                strm.avail_out = ress->dstBufferSize;
-    }   }   }
+            }
+        }
+        if (ret == Z_STREAM_END) break;
+    }
 
+    if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+    ress->srcBufferLoaded = strm.avail_in;
     inflateEnd(&strm);
     return outFileSize;
 }
@@ -717,19 +730,19 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
     for ( ; ; ) {
         /* check magic number -> version */
         size_t const toRead = 4;
-        size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
         const BYTE* buf = (const BYTE*)ress.srcBuffer;
-        if (sizeCheck==0) {
+        if (ress.srcBufferLoaded < toRead)
+            ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
+        if (ress.srcBufferLoaded==0) {
             if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
             break;   /* no more input */
         }
         readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
-        if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
+        if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
         if (buf[0] == 31 && buf[1] == 139) { /* gz header */
 #ifdef ZSTD_GZDECOMPRESS
-            unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead);
+            unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
             if (result == 0) return 1;
-            printf("gzip=%d\n", (int)result);
             filesize += result;
 #else
             DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
@@ -738,8 +751,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
         } else {
             if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
                 if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
-                    unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
+                    unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
-                    printf("pass-through=%d\n", (int)result);
                     if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName);  /* error should never happen */
                     return result;
                 } else {
@@ -747,8 +759,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
                     fclose(srcFile);
                     return 1;
             }   }
-            filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize);
+            filesize += FIO_decompressFrame(&ress, srcFile, filesize);
-            printf("zstd filesize=%d\n", (int)filesize);
         }
     }
 

tests/Makefile

@@ -37,10 +37,10 @@ FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
 ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
-ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
+ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 
 
@@ -53,7 +53,7 @@ EXT =
 endif
 
 VOID = /dev/null
-ZBUFFTEST = -T2mn
+ZSTREAM_TESTTIME = -T2mn
 FUZZERTEST= -T5mn
 ZSTDRTTEST= --test-large-data
 
@@ -61,9 +61,9 @@ ZSTDRTTEST= --test-large-data
 
 default: fullbench
 
-all: fullbench fuzzer zstreamtest paramgrill datagen
+all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
 
-all32: fullbench32 fuzzer32 zstreamtest32
+all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
 
 
 
@@ -93,19 +93,21 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 
-fuzzer  : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
 fuzzer  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
 fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
 	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
 
-zbufftest  : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+zbufftest : CFLAGS += -Wno-deprecated-declarations
+zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)   # flag required to silence deprecation warnings
 
-zbufftest32  : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
+zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 zstreamtest  : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
@@ -194,9 +196,20 @@ test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
 test-zstd-nolegacy: zstd-nolegacy zstd-playTests
 
 test-gzstd: gzstd
+	$(PRGDIR)/zstd README.md test-zstd-speed.py
 	gzip README.md test-zstd-speed.py
+	cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst
+	cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz
 	$(PRGDIR)/zstd -d README.md.gz -o README2.md
 	$(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz
+	$(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz
+	diff -q zstd_gz gz_zstd
+	echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
+	echo Hello World GZIP | gzip -c - >hello.gz
+	echo Hello World TEXT >hello.txt
+	cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz
+	$(PRGDIR)/zstd -dcf hello.*
+	$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
 
 test-fullbench: fullbench datagen
 	$(QEMU_SYS) ./fullbench -i1
@@ -213,15 +226,15 @@ test-fuzzer32: fuzzer32
 	$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST)
 
 test-zbuff: zbufftest
-	$(QEMU_SYS) ./zbufftest $(ZBUFFTEST)
+	$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
 
 test-zbuff32: zbufftest32
-	$(QEMU_SYS) ./zbufftest32 $(ZBUFFTEST)
+	$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
 
 test-zstream: zstreamtest
-	$(QEMU_SYS) ./zstreamtest $(ZBUFFTEST)
+	$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME)
 
 test-zstream32: zstreamtest32
-	$(QEMU_SYS) ./zstreamtest32 $(ZBUFFTEST)
+	$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
 
 endif

tests/fuzzer.c

@@ -27,7 +27,7 @@
 #include <time.h>         /* clock_t */
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "zstd.h"         /* ZSTD_VERSION_STRING */
-#include "zstd_errors.h" /* ZSTD_getErrorCode */
+#include "zstd_errors.h"  /* ZSTD_getErrorCode */
 #include "zdict.h"        /* ZDICT_trainFromBuffer */
 #include "datagen.h"      /* RDG_genBuffer */
 #include "mem.h"
@@ -246,6 +246,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         size_t const sampleUnitSize = 8 KB;
         U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
         size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
+        U32 dictID;
 
         if (dictBuffer==NULL || samplesSizes==NULL) {
             free(dictBuffer);
@@ -261,10 +262,9 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
 
         DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
-        {   U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize);
+        dictID = ZDICT_getDictID(dictBuffer, dictSize);
-            if (dictID==0) goto _output_error;
+        if (dictID==0) goto _output_error;
-            DISPLAYLEVEL(4, "OK : %u \n", dictID);
+        DISPLAYLEVEL(4, "OK : %u \n", dictID);
-        }
 
         DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
         cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
@@ -273,6 +273,18 @@ static int basicUnitTests(U32 seed, double compressibility)
         if (ZSTD_isError(cSize)) goto _output_error;
         DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
 
+        DISPLAYLEVEL(4, "test%3i : retrieve dictID from dictionary : ", testNb++);
+        {   U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
+            if (did != dictID) goto _output_error;   /* non-conformant (content-only) dictionary */
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
+        DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
+        {   U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
+            if (did != dictID) goto _output_error;   /* non-conformant (content-only) dictionary */
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
         DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
         CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
                                        decodedBuffer, CNBuffSize,

zlibWrapper/.gitignore

@@ -2,11 +2,14 @@
 _*
 example.*
 example_zstd.*
+example_gz.*
 fitblk.*
 fitblk_zstd.*
 zwrapbench.*
 foo.gz
 
+minigzip
+minigzip_zstd
 example
 example_zstd
 fitblk

zlibWrapper/Makefile

@@ -18,7 +18,7 @@ EXAMPLE_PATH = examples
 PROGRAMS_PATH = ../programs
 TEST_FILE = ../doc/zstd_compression_format.md
 
-CPPFLAGS = -DXXH_NAMESPACE=XXH_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
+CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
 CFLAGS  ?= $(MOREFLAGS) -O3 -std=gnu99
 CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wstrict-aliasing=1