Merge pull request #78 from Cyan4973/newAPI

New api

Makefile

@@ -32,12 +32,19 @@
 # ################################################################
 
 # Version number
-export VERSION := 0.3.6
+export VERSION := 0.4.0
 
 PRGDIR  = programs
 ZSTDDIR = lib
 
-.PHONY: clean
+# Define nul output
+ifneq (,$(filter Windows%,$(OS)))
+VOID = nul
+else
+VOID = /dev/null
+endif
+
+.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
 
 default: zstdprogram
 
@@ -49,8 +56,8 @@ zstdprogram:
 	$(MAKE) -C $(PRGDIR)
 
 clean:
-	$(MAKE) -C $(ZSTDDIR) $@
+	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
-	$(MAKE) -C $(PRGDIR) $@
+	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@echo Cleaning completed
 
 

NEWS

@@ -1,3 +1,9 @@
+v0.4.0
+Command line utility is now compatible with high compression levels 
+Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
+Rolling buffer support
+
 v0.3.6
 small blocks params
 

lib/Makefile

@@ -27,8 +27,8 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 # 
 # You can contact the author at :
+#  - ZSTD homepage : http://www.zstd.net
 #  - ZSTD source repository : https://github.com/Cyan4973/zstd
-#  - Public forum : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
 # Version numbers
@@ -63,14 +63,17 @@ else
 	SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
-default: libzstd
 
-all: libzstd
+.PHONY: default all clean install uninstall
 
-libzstd: zstd.c huff0.c fse.c
+default: clean libzstd
+
+all: clean libzstd
+
+libzstd: zstd_compress.c zstd_decompress.c huff0.c fse.c legacy/zstd_v01.c legacy/zstd_v02.c
 	@echo compiling static library
 	@$(CC) $(FLAGS) -c $^
-	@$(AR) rcs libzstd.a zstd.o huff0.o fse.o
+	@$(AR) rcs libzstd.a *.o
 	@echo compiling dynamic library $(LIBVER)
 	@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
 	@echo creating versioned links

lib/error.h

@@ -68,6 +68,7 @@ extern "C" {
 
 #define ERROR_LIST(ITEM) \
         ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
+        ITEM(PREFIX(mode_unsupported)) ITEM(PREFIX(init_missing))\
         ITEM(PREFIX(memory_allocation)) \
         ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
         ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \

lib/legacy/zstd_v02.c

@@ -475,7 +475,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 MEM_STATIC unsigned BIT_highbit32 (register U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
-    unsigned long r;
+    unsigned long r=0;
     _BitScanReverse ( &r, val );
     return (unsigned) r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */

lib/mem.h

@@ -49,12 +49,12 @@ extern "C" {
 /******************************************
 *  Compiler-specific
 ******************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#if defined(__GNUC__)
+#  define MEM_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #  define MEM_STATIC static inline
 #elif defined(_MSC_VER)
 #  define MEM_STATIC static __inline
-#elif defined(__GNUC__)
-#  define MEM_STATIC static __attribute__((unused))
 #else
 #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif

lib/zstd.h

@@ -47,8 +47,8 @@ extern "C" {
 *  Version
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
-#define ZSTD_VERSION_MINOR    3    /* for new (non-breaking) interface capabilities */
+#define ZSTD_VERSION_MINOR    4    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  6    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 
@@ -57,7 +57,8 @@ unsigned ZSTD_versionNumber (void);
 *  Simple functions
 ***************************************/
 size_t ZSTD_compress(   void* dst, size_t maxDstSize,
-                  const void* src, size_t srcSize);
+                  const void* src, size_t srcSize,
+                         int  compressionLevel);
 
 size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
                   const void* src, size_t compressedSize);
@@ -100,7 +101,7 @@ size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 ZSTD_compressCCtx() :
     Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated
 */
-size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
 
 
 #if defined (__cplusplus)

lib/zstd_buffered.c

@@ -0,0 +1,542 @@
+/*
+    Buffered version of Zstd compression library
+    Copyright (C) 2015, Yann Collet.
+
+    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+    - zstd source repository : https://github.com/Cyan4973/zstd
+    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
+/* *************************************
+*  Includes
+***************************************/
+#include <stdlib.h>
+#include "error.h"
+#include "zstd_static.h"
+#include "zstd_buffered_static.h"
+
+
+/** ************************************************
+*  Streaming compression
+*
+*  A ZBUFF_CCtx object is required to track streaming operation.
+*  Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
+*  Use ZBUFF_compressInit() to start a new compression operation.
+*  ZBUFF_CCtx objects can be reused multiple times.
+*
+*  Use ZBUFF_compressContinue() repetitively to consume your input.
+*  *srcSizePtr and *maxDstSizePtr can be any size.
+*  The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
+*  Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
+*  The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
+*  Note that it will not output more than *maxDstSizePtr.
+*  Therefore, some content might still be left into its internal buffer if dst buffer is too small.
+*  @return : nb of bytes still present into internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  ZBUFF_compressEnd() instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
+*  @return : nb of bytes still present into internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  Hint : recommended buffer sizes (not compulsory)
+*  input : 128 KB block size is the internal unit, it improves latency to use this value.
+*  output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed.
+* **************************************************/
+
+typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
+
+/* *** Ressources *** */
+struct ZBUFF_CCtx_s {
+    ZSTD_CCtx* zc;
+    char* inBuff;
+    size_t inBuffSize;
+    size_t inToCompress;
+    size_t inBuffPos;
+    size_t inBuffTarget;
+    size_t blockSize;
+    char* outBuff;
+    size_t outBuffSize;
+    size_t outBuffContentSize;
+    size_t outBuffFlushedSize;
+    ZBUFF_cStage stage;
+};   /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
+
+ZBUFF_CCtx* ZBUFF_createCCtx(void)
+{
+    ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
+    if (zbc==NULL) return NULL;
+    memset(zbc, 0, sizeof(*zbc));
+    zbc->zc = ZSTD_createCCtx();
+    return zbc;
+}
+
+size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
+{
+    if (zbc==NULL) return 0;   /* support free on NULL */
+    ZSTD_freeCCtx(zbc->zc);
+    free(zbc->inBuff);
+    free(zbc->outBuff);
+    free(zbc);
+    return 0;
+}
+
+
+/* *** Initialization *** */
+
+#define MIN(a,b)    ( ((a)<(b)) ? (a) : (b) )
+#define BLOCKSIZE   (128 * 1024)   /* a bit too "magic", should come from reference */
+size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params)
+{
+    size_t neededInBuffSize;
+
+    ZSTD_validateParams(&params);
+    neededInBuffSize = (size_t)1 << params.windowLog;
+
+    /* allocate buffers */
+    if (zbc->inBuffSize < neededInBuffSize)
+    {
+        zbc->inBuffSize = neededInBuffSize;
+        free(zbc->inBuff);   /* should not be necessary */
+        zbc->inBuff = (char*)malloc(neededInBuffSize);
+        if (zbc->inBuff == NULL) return ERROR(memory_allocation);
+    }
+    zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize);
+    if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1)
+    {
+        zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
+        free(zbc->outBuff);   /* should not be necessary */
+        zbc->outBuff = (char*)malloc(zbc->outBuffSize);
+        if (zbc->outBuff == NULL) return ERROR(memory_allocation);
+    }
+
+    zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params);
+    if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
+
+    zbc->inToCompress = 0;
+    zbc->inBuffPos = 0;
+    zbc->inBuffTarget = zbc->blockSize;
+    zbc->outBuffFlushedSize = 0;
+    zbc->stage = ZBUFFcs_flush;   /* starts by flushing the header */
+    return 0;   /* ready to go */
+}
+
+size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
+{
+    return ZBUFF_compressInit_advanced(zbc, ZSTD_getParams(compressionLevel, 0));
+}
+
+
+
+/* *** Compression *** */
+
+static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    size_t length = MIN(maxDstSize, srcSize);
+    memcpy(dst, src, length);
+    return length;
+}
+
+static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
+                              void* dst, size_t* maxDstSizePtr,
+                        const void* src, size_t* srcSizePtr,
+                              int flush)   /* aggregate : wait for full block before compressing */
+{
+    U32 notDone = 1;
+    const char* const istart = (const char*)src;
+    const char* ip = istart;
+    const char* const iend = istart + *srcSizePtr;
+    char* const ostart = (char*)dst;
+    char* op = ostart;
+    char* const oend = ostart + *maxDstSizePtr;
+
+    while (notDone)
+    {
+        switch(zbc->stage)
+        {
+        case ZBUFFcs_init: return ERROR(init_missing);   /* call ZBUFF_compressInit() first ! */
+
+        case ZBUFFcs_load:
+            /* complete inBuffer */
+            {
+                size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos;
+                size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
+                zbc->inBuffPos += loaded;
+                ip += loaded;
+                if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) )
+                    { notDone = 0; break; }  /* not enough input to get a full block : stop there, wait for more */
+            }
+            /* compress current block (note : this stage cannot be stopped in the middle) */
+            {
+                void* cDst;
+                size_t cSize;
+                size_t iSize = zbc->inBuffPos - zbc->inToCompress;
+                if ((size_t)(oend-op) > ZSTD_compressBound(iSize))
+                    cDst = op;   /* compress directly into output buffer (avoid flush stage) */
+                else
+                    cDst = zbc->outBuff;
+                cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize);
+                if (ZSTD_isError(cSize)) return cSize;
+                /* prepare next block */
+                zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
+                if (zbc->inBuffTarget > zbc->inBuffSize)
+                    { zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; }   /* note : inBuffSize >= blockSize */
+                zbc->inToCompress = zbc->inBuffPos;
+                if (cDst == op) { op += cSize; break; }   /* no need to flush */
+                zbc->outBuffContentSize = cSize;
+                zbc->outBuffFlushedSize = 0;
+                zbc->stage = ZBUFFcs_flush;
+                // break;   /* flush stage follows */
+            }
+
+        case ZBUFFcs_flush:
+            /* flush into dst */
+            {
+                size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+                size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
+                op += flushed;
+                zbc->outBuffFlushedSize += flushed;
+                if (toFlush!=flushed)
+                    { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
+                zbc->outBuffContentSize = 0;
+                zbc->outBuffFlushedSize = 0;
+                zbc->stage = ZBUFFcs_load;
+                break;
+            }
+        }
+    }
+
+    *srcSizePtr = ip - istart;
+    *maxDstSizePtr = op - ostart;
+    {
+        size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
+        if (hintInSize==0) hintInSize = zbc->blockSize;
+        return hintInSize;
+    }
+}
+
+size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
+                              void* dst, size_t* maxDstSizePtr,
+                        const void* src, size_t* srcSizePtr)
+{ return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); }
+
+
+
+/* *** Finalize *** */
+
+size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
+{
+    size_t srcSize = 0;
+    ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, NULL, &srcSize, 1);
+    return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+}
+
+
+size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
+{
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* op = ostart;
+    BYTE* const oend = ostart + *maxDstSizePtr;
+    size_t outSize = *maxDstSizePtr;
+    size_t epilogueSize, remaining;
+    ZBUFF_compressFlush(zbc, dst, &outSize);    /* flush any remaining inBuff */
+    op += outSize;
+    epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize);   /* epilogue into outBuff */
+    zbc->outBuffContentSize += epilogueSize;
+    outSize = oend-op;
+    zbc->stage = ZBUFFcs_flush;
+    remaining = ZBUFF_compressFlush(zbc, op, &outSize);   /* attempt to flush epilogue into dst */
+    op += outSize;
+    if (!remaining) zbc->stage = ZBUFFcs_init;  /* close only if nothing left to flush */
+    *maxDstSizePtr = op-ostart;                 /* tells how many bytes were written */
+    return remaining;
+}
+
+
+
+/** ************************************************
+*  Streaming decompression
+*
+*  A ZBUFF_DCtx object is required to track streaming operation.
+*  Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
+*  Use ZBUFF_decompressInit() to start a new decompression operation.
+*  ZBUFF_DCtx objects can be reused multiple times.
+*
+*  Use ZBUFF_decompressContinue() repetitively to consume your input.
+*  *srcSizePtr and *maxDstSizePtr can be any size.
+*  The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
+*  Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
+*  The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+*            or 0 when a frame is completely decoded
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  Hint : recommended buffer sizes (not compulsory)
+*  output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
+*  input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* **************************************************/
+
+typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
+               ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
+
+/* *** Resource management *** */
+
+#define ZSTD_frameHeaderSize_max 5   /* too magical, should come from reference */
+struct ZBUFF_DCtx_s {
+    ZSTD_DCtx* zc;
+    ZSTD_parameters params;
+    char* inBuff;
+    size_t inBuffSize;
+    size_t inPos;
+    char* outBuff;
+    size_t outBuffSize;
+    size_t outStart;
+    size_t outEnd;
+    size_t hPos;
+    ZBUFF_dStage stage;
+    unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
+};   /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
+
+
+ZBUFF_DCtx* ZBUFF_createDCtx(void)
+{
+    ZBUFF_DCtx* zbc = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
+    if (zbc==NULL) return NULL;
+    memset(zbc, 0, sizeof(*zbc));
+    zbc->zc = ZSTD_createDCtx();
+    zbc->stage = ZBUFFds_init;
+    return zbc;
+}
+
+size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
+{
+    if (zbc==NULL) return 0;   /* support free on null */
+    ZSTD_freeDCtx(zbc->zc);
+    free(zbc->inBuff);
+    free(zbc->outBuff);
+    free(zbc);
+    return 0;
+}
+
+
+
+/* *** Initialization *** */
+
+size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
+{
+    zbc->stage = ZBUFFds_readHeader;
+    zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
+    return ZSTD_resetDCtx(zbc->zc);
+}
+
+
+
+/* *** Decompression *** */
+
+size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
+{
+    const char* const istart = (const char*)src;
+    const char* ip = istart;
+    const char* const iend = istart + *srcSizePtr;
+    char* const ostart = (char*)dst;
+    char* op = ostart;
+    char* const oend = ostart + *maxDstSizePtr;
+    U32 notDone = 1;
+
+    while (notDone)
+    {
+        switch(zbc->stage)
+        {
+
+        case ZBUFFds_init :
+            return ERROR(init_missing);
+
+        case ZBUFFds_readHeader :
+            /* read header from src */
+            {
+                size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
+                if (ZSTD_isError(headerSize)) return headerSize;
+                if (headerSize)
+                {
+                    /* not enough input to decode header : tell how many bytes would be necessary */
+                    memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
+                    zbc->hPos += *srcSizePtr;
+                    *maxDstSizePtr = 0;
+                    zbc->stage = ZBUFFds_loadHeader;
+                    return headerSize - zbc->hPos;
+                }
+                zbc->stage = ZBUFFds_decodeHeader;
+                break;
+            }
+
+        case ZBUFFds_loadHeader:
+            /* complete header from src */
+            {
+                size_t headerSize = ZBUFF_limitCopy(
+                    zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos,
+                    src, *srcSizePtr);
+                zbc->hPos += headerSize;
+                ip += headerSize;
+                headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos);
+                if (ZSTD_isError(headerSize)) return headerSize;
+                if (headerSize)
+                {
+                    /* not enough input to decode header : tell how many bytes would be necessary */
+                    *maxDstSizePtr = 0;
+                    return headerSize - zbc->hPos;
+                }
+                // zbc->stage = ZBUFFds_decodeHeader; break;   /* useless : stage follows */
+            }
+
+        case ZBUFFds_decodeHeader:
+                /* apply header to create / resize buffers */
+                {
+                    size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
+                    size_t neededInSize = BLOCKSIZE;   /* a block is never > BLOCKSIZE */
+                    if (zbc->inBuffSize < neededInSize)
+                    {
+                        free(zbc->inBuff);
+                        zbc->inBuffSize = neededInSize;
+                        zbc->inBuff = (char*)malloc(neededInSize);
+                        if (zbc->inBuff == NULL) return ERROR(memory_allocation);
+                    }
+                    if (zbc->outBuffSize < neededOutSize)
+                    {
+                        free(zbc->outBuff);
+                        zbc->outBuffSize = neededOutSize;
+                        zbc->outBuff = (char*)malloc(neededOutSize);
+                        if (zbc->outBuff == NULL) return ERROR(memory_allocation);
+                    }
+                }
+                if (zbc->hPos)
+                {
+                    /* some data already loaded into headerBuffer : transfer into inBuff */
+                    memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
+                    zbc->inPos = zbc->hPos;
+                    zbc->hPos = 0;
+                    zbc->stage = ZBUFFds_load;
+                    break;
+                }
+                zbc->stage = ZBUFFds_read;
+
+        case ZBUFFds_read:
+            {
+                size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
+                if (neededInSize==0)   /* end of frame */
+                {
+                    zbc->stage = ZBUFFds_init;
+                    notDone = 0;
+                    break;
+                }
+                if ((size_t)(iend-ip) >= neededInSize)
+                {
+                    /* directly decode from src */
+                    size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
+                        zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
+                        ip, neededInSize);
+                    if (ZSTD_isError(decodedSize)) return decodedSize;
+                    ip += neededInSize;
+                    if (!decodedSize) break;   /* this was just a header */
+                    zbc->outEnd = zbc->outStart +  decodedSize;
+                    zbc->stage = ZBUFFds_flush;
+                    break;
+                }
+                if (ip==iend) { notDone = 0; break; }   /* no more input */
+                zbc->stage = ZBUFFds_load;
+            }
+
+        case ZBUFFds_load:
+            {
+                size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
+                size_t toLoad = neededInSize - zbc->inPos;   /* should always be <= remaining space within inBuff */
+                size_t loadedSize;
+                if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected);   /* should never happen */
+                loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
+                ip += loadedSize;
+                zbc->inPos += loadedSize;
+                if (loadedSize < toLoad) { notDone = 0; break; }   /* not enough input, wait for more */
+                {
+                    size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
+                        zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
+                        zbc->inBuff, neededInSize);
+                    if (ZSTD_isError(decodedSize)) return decodedSize;
+                    zbc->inPos = 0;   /* input is consumed */
+                    if (!decodedSize) { zbc->stage = ZBUFFds_read; break; }   /* this was just a header */
+                    zbc->outEnd = zbc->outStart +  decodedSize;
+                    zbc->stage = ZBUFFds_flush;
+                    // break; /* ZBUFFds_flush follows */
+                }
+            }
+        case ZBUFFds_flush:
+            {
+                size_t toFlushSize = zbc->outEnd - zbc->outStart;
+                size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
+                op += flushedSize;
+                zbc->outStart += flushedSize;
+                if (flushedSize == toFlushSize)
+                {
+                    zbc->stage = ZBUFFds_read;
+                    if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
+                        zbc->outStart = zbc->outEnd = 0;
+                }
+                break;
+            }
+        }
+    }
+
+    *srcSizePtr = ip-istart;
+    *maxDstSizePtr = op-ostart;
+
+    return ZSTD_nextSrcSizeToDecompress(zbc->zc) - zbc->inPos;
+}
+
+
+
+
+
+
+
+
+/* *************************************
+*  Tool functions
+***************************************/
+unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
+const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+
+size_t ZBUFF_recommendedCInSize()  { return BLOCKSIZE; }
+size_t ZBUFF_recommendedCOutSize() { return ZSTD_compressBound(BLOCKSIZE) + 6; }
+size_t ZBUFF_recommendedDInSize()  { return BLOCKSIZE + 3; }
+size_t ZBUFF_recommendedDOutSize() { return BLOCKSIZE; }

lib/zstd_buffered.h

@@ -0,0 +1,146 @@
+/*
+    Buffered version of Zstd compression library
+    Copyright (C) 2015, Yann Collet.
+
+    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+    - zstd source repository : https://github.com/Cyan4973/zstd
+    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#ifndef ZSTD_BUFFERED_H
+#define ZSTD_BUFFERED_H
+
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* *************************************
+*  Includes
+***************************************/
+#include <stddef.h>   /* size_t */
+
+
+/* *************************************
+*  Streaming functions
+***************************************/
+typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
+ZBUFF_CCtx* ZBUFF_createCCtx(void);
+size_t      ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
+
+size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
+size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
+size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
+size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
+
+/** ************************************************
+*  Streaming compression
+*
+*  A ZBUFF_CCtx object is required to track streaming operation.
+*  Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
+*  Use ZBUFF_compressInit() to start a new compression operation.
+*  ZBUFF_CCtx objects can be reused multiple times.
+*
+*  Use ZBUFF_compressContinue() repetitively to consume input stream.
+*  *srcSizePtr and *maxDstSizePtr can be any size.
+*  The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
+*  Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
+*  The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or move dst .
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
+*  Note that it will not output more than *maxDstSizePtr.
+*  Therefore, some content might still be left into its internal buffer if dst buffer is too small.
+*  @return : nb of bytes still present into internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  ZBUFF_compressEnd() instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
+*  In which case, call again ZBUFF_compressFlush() to complete the flush.
+*  @return : nb of bytes still present into internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize
+*  input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value.
+*  output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
+*  By using both, you ensure that input will be entirely consumed, and output will always contain the result.
+* **************************************************/
+
+
+typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
+ZBUFF_DCtx* ZBUFF_createDCtx(void);
+size_t      ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
+
+size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
+size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
+
+/** ************************************************
+*  Streaming decompression
+*
+*  A ZBUFF_DCtx object is required to track streaming operation.
+*  Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
+*  Use ZBUFF_decompressInit() to start a new decompression operation.
+*  ZBUFF_DCtx objects can be reused multiple times.
+*
+*  Use ZBUFF_decompressContinue() repetitively to consume your input.
+*  *srcSizePtr and *maxDstSizePtr can be any size.
+*  The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
+*  Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
+*  The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst.
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+*            or 0 when a frame is completely decoded
+*            or an error code, which can be tested using ZBUFF_isError().
+*
+*  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
+*  output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
+*  input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* **************************************************/
+
+
+/* *************************************
+*  Tool functions
+***************************************/
+unsigned ZBUFF_isError(size_t errorCode);
+const char* ZBUFF_getErrorName(size_t errorCode);
+
+/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
+*   These sizes are not compulsory, they just tend to offer better latency */
+size_t ZBUFF_recommendedCInSize(void);
+size_t ZBUFF_recommendedCOutSize(void);
+size_t ZBUFF_recommendedDInSize(void);
+size_t ZBUFF_recommendedDOutSize(void);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif  /* ZSTD_BUFFERED_H */

lib/zstd_buffered_static.h

@@ -1,6 +1,6 @@
 /*
-    zstdhc - high compression variant
+    zstd - buffered version of compression library
-    Header File
+    experimental complementary API, for static linking only
     Copyright (C) 2015, Yann Collet.
 
     BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -27,9 +27,16 @@
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd source repository : http://www.zstd.net
+    - zstd source repository : https://github.com/Cyan4973/zstd
+    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
 */
-#pragma once
+#ifndef ZSTD_BUFFERED_STATIC_H
+#define ZSTD_BUFFERED_STATIC_H
+
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
 
 #if defined (__cplusplus)
 extern "C" {
@@ -38,39 +45,18 @@ extern "C" {
 /* *************************************
 *  Includes
 ***************************************/
-#include <stddef.h>   /* size_t */
+#include "zstd_static.h"
+#include "zstd_buffered.h"
 
 
 /* *************************************
-*  Simple function
+*  Advanced Streaming functions
 ***************************************/
-/**
+size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params);
-ZSTD_HC_compress() :
-    Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
-    Destination buffer must be already allocated.
-    Compression runs faster if maxDstSize >=  ZSTD_compressBound(srcSize).
-    @return : the number of bytes written into buffer 'dst'
-              or an error code if it fails (which can be tested using ZSTD_isError())
-*/
-size_t ZSTD_HC_compress(void* dst, size_t maxDstSize,
-                  const void* src, size_t srcSize,
-                  int compressionLevel);
-
-
-/* *************************************
-*  Advanced functions
-***************************************/
-typedef struct ZSTD_HC_CCtx_s ZSTD_HC_CCtx;   /* incomplete type */
-ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void);
-size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx);
-
-/**
-ZSTD_HC_compressCCtx() :
-    Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated
-*/
-size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
 
 
 #if defined (__cplusplus)
 }
 #endif
+
+#endif  /* ZSTD_BUFFERED_STATIC_H */

lib/zstd_decompress.c

@@ -0,0 +1,873 @@
+/*
+    zstd - standard compression library
+    Copyright (C) 2014-2015, Yann Collet.
+
+    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+    - zstd source repository : https://github.com/Cyan4973/zstd
+    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/* ***************************************************************
+*  Tuning parameters
+*****************************************************************/
+/*!
+ * HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0, fastest), or in memory heap (1, requires malloc())
+ * Note that compression context is fairly large, as a consequence heap memory is recommended.
+ */
+#ifndef ZSTD_HEAPMODE
+#  define ZSTD_HEAPMODE 1
+#endif /* ZSTD_HEAPMODE */
+
+/*!
+*  LEGACY_SUPPORT :
+*  ZSTD_decompress() can decode older formats (starting from zstd 0.1+)
+*/
+#ifndef ZSTD_LEGACY_SUPPORT
+#  define ZSTD_LEGACY_SUPPORT 1
+#endif
+
+
+/* *******************************************************
+*  Includes
+*********************************************************/
+#include <stdlib.h>      /* calloc */
+#include <string.h>      /* memcpy, memmove */
+#include <stdio.h>       /* debug : printf */
+#include "mem.h"         /* low level memory routines */
+#include "zstd_static.h"
+#include "zstd_internal.h"
+#include "fse_static.h"
+#include "huff0.h"
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+#  include "zstd_legacy.h"
+#endif
+
+/* *******************************************************
+*  Compiler specifics
+*********************************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  define FORCE_INLINE static __forceinline
+#  include <intrin.h>                    /* For Visual 2005 */
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#else
+#  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#  ifdef __GNUC__
+#    define FORCE_INLINE static inline __attribute__((always_inline))
+#  else
+#    define FORCE_INLINE static inline
+#  endif
+#endif
+
+
+/* *************************************
+*  Local types
+***************************************/
+typedef struct
+{
+    blockType_t blockType;
+    U32 origSize;
+} blockProperties_t;
+
+
+/* *******************************************************
+*  Memory operations
+**********************************************************/
+static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
+
+
+/* *************************************
+*  Error Management
+***************************************/
+unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
+
+/*! ZSTD_isError
+*   tells if a return value is an error code */
+unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
+
+/*! ZSTD_getErrorName
+*   provides error code string (useful for debugging) */
+const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
+
+/* *************************************************************
+*   Context management
+***************************************************************/
+typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
+               ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
+
+struct ZSTD_DCtx_s
+{
+    U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
+    U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+    U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+    void* previousDstEnd;
+    void* base;
+    void* vBase;
+    void* dictEnd;
+    size_t expected;
+    size_t headerSize;
+    ZSTD_parameters params;
+    blockType_t bType;
+    ZSTD_dStage stage;
+    const BYTE* litPtr;
+    size_t litBufSize;
+    size_t litSize;
+    BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
+    BYTE headerBuffer[ZSTD_frameHeaderSize_max];
+};   /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
+
+size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
+{
+    dctx->expected = ZSTD_frameHeaderSize_min;
+    dctx->stage = ZSTDds_getFrameHeaderSize;
+    dctx->previousDstEnd = NULL;
+    dctx->base = NULL;
+    dctx->vBase = NULL;
+    dctx->dictEnd = NULL;
+    return 0;
+}
+
+ZSTD_DCtx* ZSTD_createDCtx(void)
+{
+    ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx));
+    if (dctx==NULL) return NULL;
+    ZSTD_resetDCtx(dctx);
+    return dctx;
+}
+
+size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
+{
+    free(dctx);
+    return 0;
+}
+
+
+/* *************************************************************
+*   Decompression section
+***************************************************************/
+/** ZSTD_decodeFrameHeader_Part1
+*   decode the 1st part of the Frame Header, which tells Frame Header size.
+*   srcSize must be == ZSTD_frameHeaderSize_min
+*   @return : the full size of the Frame Header */
+static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize)
+{
+    U32 magicNumber;
+    if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
+    magicNumber = MEM_readLE32(src);
+    if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+    zc->headerSize = ZSTD_frameHeaderSize_min;
+    return zc->headerSize;
+}
+
+
+size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
+{
+    U32 magicNumber;
+    if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max;
+    magicNumber = MEM_readLE32(src);
+    if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+    memset(params, 0, sizeof(*params));
+    params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+    return 0;
+}
+
+/** ZSTD_decodeFrameHeader_Part2
+*   decode the full Frame Header
+*   srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1
+*   @return : 0, or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize)
+{
+    if (srcSize != zc->headerSize) return ERROR(srcSize_wrong);
+    return ZSTD_getFrameParams(&(zc->params), src, srcSize);
+}
+
+
+size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
+{
+    const BYTE* const in = (const BYTE* const)src;
+    BYTE headerFlags;
+    U32 cSize;
+
+    if (srcSize < 3) return ERROR(srcSize_wrong);
+
+    headerFlags = *in;
+    cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
+
+    bpPtr->blockType = (blockType_t)(headerFlags >> 6);
+    bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0;
+
+    if (bpPtr->blockType == bt_end) return 0;
+    if (bpPtr->blockType == bt_rle) return 1;
+    return cSize;
+}
+
+static size_t ZSTD_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
+    memcpy(dst, src, srcSize);
+    return srcSize;
+}
+
+
+/** ZSTD_decompressLiterals
+    @return : nb of bytes read from src, or an error code*/
+static size_t ZSTD_decompressLiterals(void* dst, size_t* maxDstSizePtr,
+                                const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+
+    const size_t litSize = (MEM_readLE32(src) & 0x1FFFFF) >> 2;   /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
+    const size_t litCSize = (MEM_readLE32(ip+2) & 0xFFFFFF) >> 5;   /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
+
+    if (litSize > *maxDstSizePtr) return ERROR(corruption_detected);
+    if (litCSize + 5 > srcSize) return ERROR(corruption_detected);
+
+    if (HUF_isError(HUF_decompress(dst, litSize, ip+5, litCSize))) return ERROR(corruption_detected);
+
+    *maxDstSizePtr = litSize;
+    return litCSize + 5;
+}
+
+
+/** ZSTD_decodeLiteralsBlock
+    @return : nb of bytes read from src (< srcSize ) */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
+                          const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
+{
+    const BYTE* const istart = (const BYTE*) src;
+
+    /* any compressed block with literals segment must be at least this size */
+    if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
+
+    switch(*istart & 3)
+    {
+    /* compressed */
+    case 0:
+        {
+            size_t litSize = BLOCKSIZE;
+            const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
+            dctx->litPtr = dctx->litBuffer;
+            dctx->litBufSize = BLOCKSIZE+8;
+            dctx->litSize = litSize;
+            return readSize;   /* works if it's an error too */
+        }
+    case IS_RAW:
+        {
+            const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2;   /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
+            if (litSize > srcSize-11)   /* risk of reading too far with wildcopy */
+            {
+                if (litSize > srcSize-3) return ERROR(corruption_detected);
+                memcpy(dctx->litBuffer, istart, litSize);
+                dctx->litPtr = dctx->litBuffer;
+                dctx->litBufSize = BLOCKSIZE+8;
+                dctx->litSize = litSize;
+                return litSize+3;
+            }
+            /* direct reference into compressed stream */
+            dctx->litPtr = istart+3;
+            dctx->litBufSize = srcSize-3;
+            dctx->litSize = litSize;
+            return litSize+3;        }
+    case IS_RLE:
+        {
+            const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2;   /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
+            if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
+            memset(dctx->litBuffer, istart[3], litSize);
+            dctx->litPtr = dctx->litBuffer;
+            dctx->litBufSize = BLOCKSIZE+8;
+            dctx->litSize = litSize;
+            return 4;
+        }
+    default:
+        return ERROR(corruption_detected);   /* forbidden nominal case */
+    }
+}
+
+
+size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
+                         FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
+                         const void* src, size_t srcSize)
+{
+    const BYTE* const istart = (const BYTE* const)src;
+    const BYTE* ip = istart;
+    const BYTE* const iend = istart + srcSize;
+    U32 LLtype, Offtype, MLtype;
+    U32 LLlog, Offlog, MLlog;
+    size_t dumpsLength;
+
+    /* check */
+    if (srcSize < 5) return ERROR(srcSize_wrong);
+
+    /* SeqHead */
+    *nbSeq = MEM_readLE16(ip); ip+=2;
+    LLtype  = *ip >> 6;
+    Offtype = (*ip >> 4) & 3;
+    MLtype  = (*ip >> 2) & 3;
+    if (*ip & 2)
+    {
+        dumpsLength  = ip[2];
+        dumpsLength += ip[1] << 8;
+        ip += 3;
+    }
+    else
+    {
+        dumpsLength  = ip[1];
+        dumpsLength += (ip[0] & 1) << 8;
+        ip += 2;
+    }
+    *dumpsPtr = ip;
+    ip += dumpsLength;
+    *dumpsLengthPtr = dumpsLength;
+
+    /* check */
+    if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
+
+    /* sequences */
+    {
+        S16 norm[MaxML+1];    /* assumption : MaxML >= MaxLL >= MaxOff */
+        size_t headerSize;
+
+        /* Build DTables */
+        switch(LLtype)
+        {
+        U32 max;
+        case bt_rle :
+            LLlog = 0;
+            FSE_buildDTable_rle(DTableLL, *ip++); break;
+        case bt_raw :
+            LLlog = LLbits;
+            FSE_buildDTable_raw(DTableLL, LLbits); break;
+        default :
+            max = MaxLL;
+            headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
+            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+            if (LLlog > LLFSELog) return ERROR(corruption_detected);
+            ip += headerSize;
+            FSE_buildDTable(DTableLL, norm, max, LLlog);
+        }
+
+        switch(Offtype)
+        {
+        U32 max;
+        case bt_rle :
+            Offlog = 0;
+            if (ip > iend-2) return ERROR(srcSize_wrong);   /* min : "raw", hence no header, but at least xxLog bits */
+            FSE_buildDTable_rle(DTableOffb, *ip++ & MaxOff); /* if *ip > MaxOff, data is corrupted */
+            break;
+        case bt_raw :
+            Offlog = Offbits;
+            FSE_buildDTable_raw(DTableOffb, Offbits); break;
+        default :
+            max = MaxOff;
+            headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
+            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+            if (Offlog > OffFSELog) return ERROR(corruption_detected);
+            ip += headerSize;
+            FSE_buildDTable(DTableOffb, norm, max, Offlog);
+        }
+
+        switch(MLtype)
+        {
+        U32 max;
+        case bt_rle :
+            MLlog = 0;
+            if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
+            FSE_buildDTable_rle(DTableML, *ip++); break;
+        case bt_raw :
+            MLlog = MLbits;
+            FSE_buildDTable_raw(DTableML, MLbits); break;
+        default :
+            max = MaxML;
+            headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
+            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+            if (MLlog > MLFSELog) return ERROR(corruption_detected);
+            ip += headerSize;
+            FSE_buildDTable(DTableML, norm, max, MLlog);
+        }
+    }
+
+    return ip-istart;
+}
+
+
+typedef struct {
+    size_t litLength;
+    size_t offset;
+    size_t matchLength;
+} seq_t;
+
+typedef struct {
+    BIT_DStream_t DStream;
+    FSE_DState_t stateLL;
+    FSE_DState_t stateOffb;
+    FSE_DState_t stateML;
+    size_t prevOffset;
+    const BYTE* dumps;
+    const BYTE* dumpsEnd;
+} seqState_t;
+
+
+static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
+{
+    size_t litLength;
+    size_t prevOffset;
+    size_t offset;
+    size_t matchLength;
+    const BYTE* dumps = seqState->dumps;
+    const BYTE* const de = seqState->dumpsEnd;
+
+    /* Literal length */
+    litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
+    prevOffset = litLength ? seq->offset : seqState->prevOffset;
+    if (litLength == MaxLL)
+    {
+        U32 add = *dumps++;
+        if (add < 255) litLength += add;
+        else
+        {
+            litLength = MEM_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+            dumps += 3;
+        }
+        if (dumps >= de) dumps = de-1;   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+    }
+
+    /* Offset */
+    {
+        static const U32 offsetPrefix[MaxOff+1] = {
+                1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
+                512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
+                524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
+        U32 offsetCode, nbBits;
+        offsetCode = FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream));   /* <= maxOff, by table construction */
+        if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
+        nbBits = offsetCode - 1;
+        if (offsetCode==0) nbBits = 0;   /* cmove */
+        offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits);
+        if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
+        if (offsetCode==0) offset = prevOffset;   /* cmove */
+        if (offsetCode | !litLength) seqState->prevOffset = seq->offset;   /* cmove */
+    }
+
+    /* MatchLength */
+    matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
+    if (matchLength == MaxML)
+    {
+        U32 add = *dumps++;
+        if (add < 255) matchLength += add;
+        else
+        {
+            matchLength = MEM_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+            dumps += 3;
+        }
+        if (dumps >= de) dumps = de-1;   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+    }
+    matchLength += MINMATCH;
+
+    /* save result */
+    seq->litLength = litLength;
+    seq->offset = offset;
+    seq->matchLength = matchLength;
+    seqState->dumps = dumps;
+}
+
+
+FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
+                                BYTE* const oend, seq_t sequence,
+                                const BYTE** litPtr, const BYTE* const litLimit_8,
+                                BYTE* const base, BYTE* const vBase, BYTE* const dictEnd)
+{
+    static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };   /* added */
+    static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 };   /* substracted */
+    BYTE* const oLitEnd = op + sequence.litLength;
+    const size_t sequenceLength = sequence.litLength + sequence.matchLength;
+    BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
+    BYTE* const oend_8 = oend-8;
+    const BYTE* const litEnd = *litPtr + sequence.litLength;
+    const BYTE* match = oLitEnd - sequence.offset;
+
+    /* check */
+    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
+    if (litEnd > litLimit_8) return ERROR(corruption_detected);   /* risk read beyond lit buffer */
+
+    /* copy Literals */
+    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    op = oLitEnd;
+    *litPtr = litEnd;   /* update for next sequence */
+
+    /* copy Match */
+	/* check */
+	//if (match > oLitEnd) return ERROR(corruption_detected);   /* address space overflow test (is clang optimizer wrongly removing this test ?) */
+	if (sequence.offset > (size_t)oLitEnd) return ERROR(corruption_detected);   /* address space overflow test (this test seems preserved by clang optimizer) */
+
+	if (match < base)
+	{
+		/* offset beyond prefix */
+		if (match < vBase) return ERROR(corruption_detected);
+		match = dictEnd - (base-match);
+		if (match + sequence.matchLength <= dictEnd)
+		{
+			memcpy(oLitEnd, match, sequence.matchLength);
+			return sequenceLength;
+		}
+		/* span extDict & currentPrefixSegment */
+		{
+			size_t length1 = dictEnd - match;
+			memcpy(oLitEnd, match, length1);
+			op = oLitEnd + length1;
+			sequence.matchLength -= length1;
+			match = base;
+		}
+	}
+
+	/* match within prefix */
+	if (sequence.offset < 8)
+	{
+		/* close range match, overlap */
+		const int sub2 = dec64table[sequence.offset];
+		op[0] = match[0];
+		op[1] = match[1];
+		op[2] = match[2];
+		op[3] = match[3];
+		match += dec32table[sequence.offset];
+		ZSTD_copy4(op+4, match);
+		match -= sub2;
+	}
+	else
+	{
+		ZSTD_copy8(op, match);
+	}
+	op += 8; match += 8;
+
+	if (oMatchEnd > oend-12)
+	{
+		if (op < oend_8)
+		{
+			ZSTD_wildcopy(op, match, oend_8 - op);
+			match += oend_8 - op;
+			op = oend_8;
+		}
+		while (op < oMatchEnd) *op++ = *match++;
+	}
+	else
+	{
+		ZSTD_wildcopy(op, match, sequence.matchLength-8);   /* works even if matchLength < 8 */
+	}
+	return sequenceLength;
+}
+
+
+static size_t ZSTD_decompressSequences(
+                               ZSTD_DCtx* dctx,
+                               void* dst, size_t maxDstSize,
+                         const void* seqStart, size_t seqSize)
+{
+    const BYTE* ip = (const BYTE*)seqStart;
+    const BYTE* const iend = ip + seqSize;
+    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* op = ostart;
+    BYTE* const oend = ostart + maxDstSize;
+    size_t errorCode, dumpsLength;
+    const BYTE* litPtr = dctx->litPtr;
+    const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
+    const BYTE* const litEnd = litPtr + dctx->litSize;
+    int nbSeq;
+    const BYTE* dumps;
+    U32* DTableLL = dctx->LLTable;
+    U32* DTableML = dctx->MLTable;
+    U32* DTableOffb = dctx->OffTable;
+    BYTE* const base = (BYTE*) (dctx->base);
+    BYTE* const vBase = (BYTE*) (dctx->vBase);
+    BYTE* const dictEnd = (BYTE*) (dctx->dictEnd);
+
+    /* Build Decoding Tables */
+    errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
+                                      DTableLL, DTableML, DTableOffb,
+                                      ip, iend-ip);
+    if (ZSTD_isError(errorCode)) return errorCode;
+    ip += errorCode;
+
+    /* Regen sequences */
+    {
+        seq_t sequence;
+        seqState_t seqState;
+
+        memset(&sequence, 0, sizeof(sequence));
+        sequence.offset = 4;
+        seqState.dumps = dumps;
+        seqState.dumpsEnd = dumps + dumpsLength;
+        seqState.prevOffset = 4;
+        errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
+        if (ERR_isError(errorCode)) return ERROR(corruption_detected);
+        FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
+        FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
+        FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
+
+        for ( ; (BIT_reloadDStream(&(seqState.DStream)) < BIT_DStream_completed) ; )
+        {
+            size_t oneSeqSize;
+            nbSeq--;
+            ZSTD_decodeSequence(&sequence, &seqState);
+            oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+            if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+            op += oneSeqSize;
+        }
+
+        if (nbSeq<0) return ERROR(corruption_detected);   /* requested too many sequences : data is corrupted */
+
+		/* now BIT_reloadDStream(&(seqState.DStream)) >= BIT_DStream_completed) */
+        for ( ; (BIT_reloadDStream(&(seqState.DStream)) == BIT_DStream_completed) && nbSeq ; )
+        {
+            size_t oneSeqSize;
+            nbSeq--;
+            ZSTD_decodeSequence(&sequence, &seqState);
+            oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+            if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+            op += oneSeqSize;
+        }
+
+        /* check if reached exact end */
+        if ( !BIT_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected);   /* DStream should be entirely and precisely consumed; otherwise data is corrupted */
+
+        /* last literal segment */
+        {
+            size_t lastLLSize = litEnd - litPtr;
+            if (litPtr > litEnd) return ERROR(corruption_detected);
+            if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
+            if (op != litPtr) memcpy(op, litPtr, lastLLSize);
+            op += lastLLSize;
+        }
+    }
+
+    return op-ostart;
+}
+
+
+static size_t ZSTD_decompressBlock(
+                            ZSTD_DCtx* dctx,
+                            void* dst, size_t maxDstSize,
+                      const void* src, size_t srcSize)
+{
+    /* blockType == blockCompressed */
+    const BYTE* ip = (const BYTE*)src;
+
+    /* Decode literals sub-block */
+    size_t litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
+    if (ZSTD_isError(litCSize)) return litCSize;
+    ip += litCSize;
+    srcSize -= litCSize;
+
+    return ZSTD_decompressSequences(dctx, dst, maxDstSize, ip, srcSize);
+}
+
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    const BYTE* ip = (const BYTE*)src;
+    const BYTE* iend = ip + srcSize;
+    BYTE* const ostart = (BYTE* const)dst;
+    BYTE* op = ostart;
+    BYTE* const oend = ostart + maxDstSize;
+    size_t remainingSize = srcSize;
+    blockProperties_t blockProperties;
+
+
+    /* init */
+    ctx->base = ctx->vBase = ctx->dictEnd = dst;
+
+    /* Frame Header */
+    {
+        size_t frameHeaderSize;
+        if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+        {
+            const U32 magicNumber = MEM_readLE32(src);
+            if (ZSTD_isLegacy(magicNumber))
+                return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
+        }
+#endif
+        frameHeaderSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
+        if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+        if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+        ip += frameHeaderSize; remainingSize -= frameHeaderSize;
+        frameHeaderSize = ZSTD_decodeFrameHeader_Part2(ctx, src, frameHeaderSize);
+        if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
+    }
+
+    /* Loop on each block */
+    while (1)
+    {
+        size_t decodedSize=0;
+        size_t cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
+        if (ZSTD_isError(cBlockSize)) return cBlockSize;
+
+        ip += ZSTD_blockHeaderSize;
+        remainingSize -= ZSTD_blockHeaderSize;
+        if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
+
+        switch(blockProperties.blockType)
+        {
+        case bt_compressed:
+            decodedSize = ZSTD_decompressBlock(ctx, op, oend-op, ip, cBlockSize);
+            break;
+        case bt_raw :
+            decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
+            break;
+        case bt_rle :
+            return ERROR(GENERIC);   /* not yet supported */
+            break;
+        case bt_end :
+            /* end of frame */
+            if (remainingSize) return ERROR(srcSize_wrong);
+            break;
+        default:
+            return ERROR(GENERIC);   /* impossible */
+        }
+        if (cBlockSize == 0) break;   /* bt_end */
+
+        if (ZSTD_isError(decodedSize)) return decodedSize;
+        op += decodedSize;
+        ip += cBlockSize;
+        remainingSize -= cBlockSize;
+    }
+
+    return op-ostart;
+}
+
+size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    ZSTD_DCtx ctx;
+    return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
+}
+
+
+/* ******************************
+*  Streaming Decompression API
+********************************/
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
+{
+    return dctx->expected;
+}
+
+size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    /* Sanity check */
+    if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
+    if (dst != ctx->previousDstEnd)   /* not contiguous */
+    {
+        ctx->dictEnd = ctx->previousDstEnd;
+        if ((dst > ctx->base) && (dst < ctx->previousDstEnd))   /* rolling buffer : new segment right into tracked memory */
+            ctx->base = (char*)dst + maxDstSize;   /* temporary affectation, for vBase calculation */
+        ctx->vBase = (char*)dst - ((char*)(ctx->dictEnd) - (char*)(ctx->base));
+        ctx->base = dst;
+    }
+
+    /* Decompress : frame header; part 1 */
+    switch (ctx->stage)
+    {
+    case ZSTDds_getFrameHeaderSize :
+        {
+            /* get frame header size */
+            if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
+            ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
+            if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize;
+            memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
+            if (ctx->headerSize > ZSTD_frameHeaderSize_min)
+            {
+                ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min;
+                ctx->stage = ZSTDds_decodeFrameHeader;
+                return 0;
+            }
+            ctx->expected = 0;   /* not necessary to copy more */
+        }
+    case ZSTDds_decodeFrameHeader:
+        {
+            /* get frame header */
+            size_t result;
+            memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected);
+            result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize);
+            if (ZSTD_isError(result)) return result;
+            ctx->expected = ZSTD_blockHeaderSize;
+            ctx->stage = ZSTDds_decodeBlockHeader;
+            return 0;
+        }
+    case ZSTDds_decodeBlockHeader:
+        {
+            /* Decode block header */
+            blockProperties_t bp;
+            size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+            if (ZSTD_isError(blockSize)) return blockSize;
+            if (bp.blockType == bt_end)
+            {
+                ctx->expected = 0;
+                ctx->stage = ZSTDds_getFrameHeaderSize;
+            }
+            else
+            {
+                ctx->expected = blockSize;
+                ctx->bType = bp.blockType;
+                ctx->stage = ZSTDds_decompressBlock;
+            }
+
+            ctx->previousDstEnd = dst;
+            return 0;
+        }
+    case 3:
+        {
+            /* Decompress : block content */
+            size_t rSize;
+            switch(ctx->bType)
+            {
+            case bt_compressed:
+                rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
+                break;
+            case bt_raw :
+                rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
+                break;
+            case bt_rle :
+                return ERROR(GENERIC);   /* not yet handled */
+                break;
+            case bt_end :   /* should never happen (filtered at phase 1) */
+                rSize = 0;
+                break;
+            default:
+                return ERROR(GENERIC);
+            }
+            ctx->stage = ZSTDds_decodeBlockHeader;
+            ctx->expected = ZSTD_blockHeaderSize;
+            ctx->previousDstEnd = (char*)dst + rSize;
+            return rSize;
+        }
+    default:
+        return ERROR(GENERIC);   /* impossible */
+    }
+}
+
+

lib/zstd_internal.h

@@ -44,123 +44,61 @@ extern "C" {
 #include "error.h"
 
 
-/* **************************************
+/* *************************************
-*  Function body to include for inlining
+*  Common macros
-****************************************/
+***************************************/
-static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
-
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
-
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
-static unsigned ZSTD_highbit(U32 val)
-{
-#   if defined(_MSC_VER)   /* Visual */
-    unsigned long r=0;
-    _BitScanReverse(&r, val);
-    return (unsigned)r;
-#   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
-    return 31 - __builtin_clz(val);
-#   else   /* Software version */
-    static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
-    U32 v = val;
-    int r;
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
-    return r;
-#   endif
-}
-
-MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
-{
-    if (MEM_isLittleEndian())
-    {
-        if (MEM_64bits())
-        {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (int)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-        }
-        else /* 32 bits */
-        {
-#       if defined(_MSC_VER)
-            unsigned long r=0;
-            _BitScanForward( &r, (U32)val );
-            return (int)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    }
-    else   /* Big Endian CPU */
-    {
-        if (MEM_32bits())
-        {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        }
-        else /* 32 bits */
-        {
-#       if defined(_MSC_VER)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        }
-    }
-}
 
 
-MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+/* *************************************
-{
+*  Common constants
-    const BYTE* const pStart = pIn;
+***************************************/
+#define ZSTD_MAGICNUMBER 0xFD2FB524   /* v0.4 */
 
-    while ((pIn<pInLimit-(sizeof(size_t)-1)))
+#define KB *(1 <<10)
-    {
+#define MB *(1 <<20)
-        size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
+#define GB *(1U<<30)
-        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
-        pIn += ZSTD_NbCommonBytes(diff);
-        return (size_t)(pIn - pStart);
-    }
 
-    if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+#define BLOCKSIZE (128 KB)                 /* define, for static allocation */
-    if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+
-    if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+static const size_t ZSTD_blockHeaderSize = 3;
-    return (size_t)(pIn - pStart);
+static const size_t ZSTD_frameHeaderSize_min = 5;
-}
+#define ZSTD_frameHeaderSize_max 5         /* define, for static allocation */
+
+#define BIT7 128
+#define BIT6  64
+#define BIT5  32
+#define BIT4  16
+#define BIT1   2
+#define BIT0   1
+
+#define IS_RAW BIT0
+#define IS_RLE BIT1
+
+#define MINMATCH 4
+#define REPCODE_STARTVALUE 4
+
+#define MLbits   7
+#define LLbits   6
+#define Offbits  5
+#define MaxML  ((1<<MLbits) - 1)
+#define MaxLL  ((1<<LLbits) - 1)
+#define MaxOff ((1<<Offbits)- 1)
+#define MLFSELog   10
+#define LLFSELog   10
+#define OffFSELog   9
+#define MaxSeq MAX(MaxLL, MaxML)
+
+#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
+#define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
+
+typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
 
 
+/* ******************************************
+*  Shared functions to include for inlining
+********************************************/
 static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
 
 #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
@@ -177,104 +115,6 @@ static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
 }
 
 
-typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
-
-typedef struct
-{
-    blockType_t blockType;
-    U32 origSize;
-} blockProperties_t;
-
-size_t ZSTD_noCompressBlock(void* op, size_t maxDstSize, const void* ip, size_t blockSize);
-
-
-typedef struct {
-    void* buffer;
-    U32*  offsetStart;
-    U32*  offset;
-    BYTE* offCodeStart;
-    BYTE* offCode;
-    BYTE* litStart;
-    BYTE* lit;
-    BYTE* litLengthStart;
-    BYTE* litLength;
-    BYTE* matchLengthStart;
-    BYTE* matchLength;
-    BYTE* dumpsStart;
-    BYTE* dumps;
-} seqStore_t;
-
-void ZSTD_resetSeqStore(seqStore_t* ssPtr);
-
-static const U32 g_searchStrength = 8;
-
-#define REPCODE_STARTVALUE 4
-#define MLbits   7
-#define LLbits   6
-#define Offbits  5
-#define MaxML  ((1<<MLbits) - 1)
-#define MaxLL  ((1<<LLbits) - 1)
-#define MaxOff   31
-
-#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
-#define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
-
-/** ZSTD_storeSeq
-    Store a sequence (literal length, literals, offset code and match length) into seqStore_t
-    @offsetCode : distance to match, or 0 == repCode
-    @matchCode : matchLength - MINMATCH
-*/
-MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
-{
-#if 0
-    static const BYTE* g_start = NULL;
-    if (g_start==NULL) g_start = literals;
-    if (literals - g_start == 8695)
-    printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
-           (U32)(literals - g_start), (U32)litLength, (U32)matchCode+4, (U32)offsetCode);
-#endif
-
-    /* copy Literals */
-    ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
-    seqStorePtr->lit += litLength;
-
-    /* literal Length */
-    if (litLength >= MaxLL)
-    {
-        *(seqStorePtr->litLength++) = MaxLL;
-        if (litLength<255 + MaxLL)
-            *(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
-        else
-        {
-            *(seqStorePtr->dumps++) = 255;
-            MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
-        }
-    }
-    else *(seqStorePtr->litLength++) = (BYTE)litLength;
-
-    /* match offset */
-    *(seqStorePtr->offset++) = (U32)offsetCode;
-
-    /* match Length */
-    if (matchCode >= MaxML)
-    {
-        *(seqStorePtr->matchLength++) = MaxML;
-        if (matchCode < 255+MaxML)
-            *(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
-        else
-        {
-            *(seqStorePtr->dumps++) = 255;
-            MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
-        }
-    }
-    else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
-}
-
-
-/* prototype, body into zstd.c */
-size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const seqStore_t* seqStorePtr, size_t srcSize);
-
-
 #if defined (__cplusplus)
 }
 #endif

lib/zstd_static.h

@@ -46,12 +46,65 @@ extern "C" {
 *  Includes
 ***************************************/
 #include "zstd.h"
+#include "mem.h"
 
 
 /* *************************************
-*  Streaming functions
+*  Types
 ***************************************/
-size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
+#define ZSTD_WINDOWLOG_MAX 26
+#define ZSTD_WINDOWLOG_MIN 18
+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
+#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
+#define ZSTD_CONTENTLOG_MIN 4
+#define ZSTD_HASHLOG_MAX 28
+#define ZSTD_HASHLOG_MIN 4
+#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN 1
+#define ZSTD_SEARCHLENGTH_MAX 7
+#define ZSTD_SEARCHLENGTH_MIN 4
+
+/** from faster to stronger */
+typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
+
+typedef struct
+{
+    U64 srcSize;       /* optional : tells how much bytes are present in the frame. Use 0 if not known. */
+    U32 windowLog;     /* largest match distance : larger == more compression, more memory needed during decompression */
+    U32 contentLog;    /* full search segment : larger == more compression, slower, more memory (useless for fast) */
+    U32 hashLog;       /* dispatch table : larger == more memory, faster */
+    U32 searchLog;     /* nb of searches : larger == more compression, slower */
+    U32 searchLength;  /* size of matches : larger == faster decompression, sometimes less compression */
+    ZSTD_strategy strategy;
+} ZSTD_parameters;
+
+
+/* *************************************
+*  Advanced function
+***************************************/
+/** ZSTD_getParams
+*   return ZSTD_parameters structure for a selected compression level and srcSize.
+*   srcSizeHint value is optional, select 0 if not known */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
+
+/** ZSTD_validateParams
+*   correct params value to remain within authorized range */
+void ZSTD_validateParams(ZSTD_parameters* params);
+
+/** ZSTD_compress_advanced
+*   Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
+size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
+                               void* dst, size_t maxDstSize,
+                         const void* src, size_t srcSize,
+                               ZSTD_parameters params);
+
+
+/* **************************************
+*  Streaming functions (bufferless mode)
+****************************************/
+size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
+
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
 
@@ -61,20 +114,91 @@ ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_resetDCtx(ZSTD_DCtx* dctx);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
+size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-/*
+/**
-  Use above functions alternatively.
+  Streaming decompression, bufferless mode
+
+  A ZSTD_DCtx object is required to track streaming operations.
+  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+  A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status.
+
+  First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+  This function doesn't consume its input. It needs enough input data to properly decode the frame header.
+  The objective is to retrieve *params.windowlog, to know how much memory is required during decoding.
+  Result : 0 if successfull, it means the ZSTD_parameters structure has been filled.
+           >0 : means there is not enough data into src. Provides the expected size to successfully decode header.
+           errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
+
+  Then it's possible to start decompression.
+  Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
   ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
-  ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
+  ZSTD_decompressContinue() will use previous data blocks during decompress.
-  Result is the number of bytes regenerated within 'dst'.
+  They should be located contiguously prior to current block. Alternatively, a round buffer is possible.
+  Just make sure that the combined of current and accessible past blocks is a minimum of (1 << windowlog).
+
+  @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'.
   It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
+
+  A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
 */
 
+
 /* *************************************
-*  Prefix - version detection
+*  Pre-defined compression levels
 ***************************************/
-#define ZSTD_magicNumber 0xFD2FB523   /* v0.3 (current)*/
+#define ZSTD_MAX_CLEVEL 20
+static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
+{   /* "default" */
+    /*   W,  C,  H,  S,  L, strat */
+    { 0, 18, 12, 12,  1,  4, ZSTD_fast    },  /* level  0 - never used */
+    { 0, 19, 13, 14,  1,  7, ZSTD_fast    },  /* level  1 */
+    { 0, 19, 15, 16,  1,  6, ZSTD_fast    },  /* level  2 */
+    { 0, 20, 18, 20,  1,  6, ZSTD_fast    },  /* level  3 */
+    { 0, 21, 19, 21,  1,  6, ZSTD_fast    },  /* level  4 */
+    { 0, 20, 14, 18,  3,  5, ZSTD_greedy  },  /* level  5 */
+    { 0, 20, 18, 19,  3,  5, ZSTD_greedy  },  /* level  6 */
+    { 0, 21, 17, 20,  3,  5, ZSTD_lazy    },  /* level  7 */
+    { 0, 21, 19, 20,  3,  5, ZSTD_lazy    },  /* level  8 */
+    { 0, 21, 20, 20,  3,  5, ZSTD_lazy2   },  /* level  9 */
+    { 0, 21, 19, 21,  4,  5, ZSTD_lazy2   },  /* level 10 */
+    { 0, 22, 20, 22,  4,  5, ZSTD_lazy2   },  /* level 11 */
+    { 0, 22, 20, 22,  5,  5, ZSTD_lazy2   },  /* level 12 */
+    { 0, 22, 21, 22,  5,  5, ZSTD_lazy2   },  /* level 13 */
+    { 0, 22, 22, 23,  5,  5, ZSTD_lazy2   },  /* level 14 */
+    { 0, 23, 23, 23,  5,  5, ZSTD_lazy2   },  /* level 15 */
+    { 0, 23, 21, 22,  5,  5, ZSTD_btlazy2 },  /* level 16 */
+    { 0, 23, 24, 23,  4,  5, ZSTD_btlazy2 },  /* level 17 */
+    { 0, 25, 24, 23,  5,  5, ZSTD_btlazy2 },  /* level 18 */
+    { 0, 25, 26, 23,  5,  5, ZSTD_btlazy2 },  /* level 19 */
+    { 0, 25, 26, 25,  6,  5, ZSTD_btlazy2 },  /* level 20 */
+},
+{   /* for srcSize <= 128 KB */
+    /* W,  C,  H,  S,  L, strat */
+    { 0, 17, 12, 12,  1,  4, ZSTD_fast    },  /* level  0 - never used */
+    { 0, 17, 12, 13,  1,  6, ZSTD_fast    },  /* level  1 */
+    { 0, 17, 15, 16,  1,  5, ZSTD_fast    },  /* level  2 */
+    { 0, 17, 16, 17,  1,  5, ZSTD_fast    },  /* level  3 */
+    { 0, 17, 13, 15,  2,  4, ZSTD_greedy  },  /* level  4 */
+    { 0, 17, 15, 17,  3,  4, ZSTD_greedy  },  /* level  5 */
+    { 0, 17, 14, 17,  3,  4, ZSTD_lazy    },  /* level  6 */
+    { 0, 17, 16, 17,  4,  4, ZSTD_lazy    },  /* level  7 */
+    { 0, 17, 16, 17,  4,  4, ZSTD_lazy2   },  /* level  8 */
+    { 0, 17, 17, 16,  5,  4, ZSTD_lazy2   },  /* level  9 */
+    { 0, 17, 17, 16,  6,  4, ZSTD_lazy2   },  /* level 10 */
+    { 0, 17, 17, 16,  7,  4, ZSTD_lazy2   },  /* level 11 */
+    { 0, 17, 17, 16,  8,  4, ZSTD_lazy2   },  /* level 12 */
+    { 0, 17, 18, 16,  4,  4, ZSTD_btlazy2 },  /* level 13 */
+    { 0, 17, 18, 16,  5,  4, ZSTD_btlazy2 },  /* level 14 */
+    { 0, 17, 18, 16,  6,  4, ZSTD_btlazy2 },  /* level 15 */
+    { 0, 17, 18, 16,  7,  4, ZSTD_btlazy2 },  /* level 16 */
+    { 0, 17, 18, 16,  8,  4, ZSTD_btlazy2 },  /* level 17 */
+    { 0, 17, 18, 16,  9,  4, ZSTD_btlazy2 },  /* level 18 */
+    { 0, 17, 18, 16, 10,  4, ZSTD_btlazy2 },  /* level 19 */
+    { 0, 17, 18, 18, 12,  4, ZSTD_btlazy2 },  /* level 20 */
+},
+};
 
 
 /* *************************************

lib/zstdhc_static.h

@@ -1,156 +0,0 @@
-/*
-    zstdhc - high compression variant
-    Header File - Experimental API, static linking only
-    Copyright (C) 2015, Yann Collet.
-
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : http://www.zstd.net
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/* *************************************
-*  Includes
-***************************************/
-#include "mem.h"
-#include "zstdhc.h"
-
-
-/* *************************************
-*  Types
-***************************************/
-/** from faster to stronger */
-typedef enum { ZSTD_HC_fast, ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazy2, ZSTD_HC_btlazy2 } ZSTD_HC_strategy;
-
-typedef struct
-{
-    U32 windowLog;     /* largest match distance : impact decompression buffer size */
-    U32 contentLog;    /* full search segment : larger == more compression, slower, more memory (useless for fast) */
-    U32 hashLog;       /* dispatch table : larger == more memory, faster*/
-    U32 searchLog;     /* nb of searches : larger == more compression, slower*/
-    U32 searchLength;  /* size of matches : larger == faster decompression */
-    ZSTD_HC_strategy strategy;
-} ZSTD_HC_parameters;
-
-/* parameters boundaries */
-#define ZSTD_HC_WINDOWLOG_MAX 26
-#define ZSTD_HC_WINDOWLOG_MIN 18
-#define ZSTD_HC_CONTENTLOG_MAX (ZSTD_HC_WINDOWLOG_MAX+1)
-#define ZSTD_HC_CONTENTLOG_MIN 4
-#define ZSTD_HC_HASHLOG_MAX 28
-#define ZSTD_HC_HASHLOG_MIN 4
-#define ZSTD_HC_SEARCHLOG_MAX (ZSTD_HC_CONTENTLOG_MAX-1)
-#define ZSTD_HC_SEARCHLOG_MIN 1
-#define ZSTD_HC_SEARCHLENGTH_MAX 7
-#define ZSTD_HC_SEARCHLENGTH_MIN 4
-
-
-/* *************************************
-*  Advanced function
-***************************************/
-/** ZSTD_HC_compress_advanced
-*   Same as ZSTD_HC_compressCCtx(), with fine-tune control of each compression parameter */
-size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
-                                 void* dst, size_t maxDstSize,
-                           const void* src, size_t srcSize,
-                                 ZSTD_HC_parameters params);
-
-/** ZSTD_HC_validateParams
-    correct params value to remain within authorized range
-    srcSizeHint value is optional, select 0 if not known */
-void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
-
-
-/* *************************************
-*  Streaming functions
-***************************************/
-size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
-size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
-
-
-/* *************************************
-*  Pre-defined compression levels
-***************************************/
-#define ZSTD_HC_MAX_CLEVEL 20
-static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[2][ZSTD_HC_MAX_CLEVEL+1] = {
-{   /* for <= 128 KB */
-    /* W,  C,  H,  S,  L, strat */
-    { 17, 12, 12,  1,  4, ZSTD_HC_fast    },  /* level  0 - never used */
-    { 17, 12, 13,  1,  6, ZSTD_HC_fast    },  /* level  1 */
-    { 17, 15, 16,  1,  5, ZSTD_HC_fast    },  /* level  2 */
-    { 17, 16, 17,  1,  5, ZSTD_HC_fast    },  /* level  3 */
-    { 17, 13, 15,  2,  4, ZSTD_HC_greedy  },  /* level  4 */
-    { 17, 15, 17,  3,  4, ZSTD_HC_greedy  },  /* level  5 */
-    { 17, 14, 17,  3,  4, ZSTD_HC_lazy    },  /* level  6 */
-    { 17, 16, 17,  4,  4, ZSTD_HC_lazy    },  /* level  7 */
-    { 17, 16, 17,  4,  4, ZSTD_HC_lazy2   },  /* level  8 */
-    { 17, 17, 16,  5,  4, ZSTD_HC_lazy2   },  /* level  9 */
-    { 17, 17, 16,  6,  4, ZSTD_HC_lazy2   },  /* level 10 */
-    { 17, 17, 16,  7,  4, ZSTD_HC_lazy2   },  /* level 11 */
-    { 17, 17, 16,  8,  4, ZSTD_HC_lazy2   },  /* level 12 */
-    { 17, 18, 16,  4,  4, ZSTD_HC_btlazy2 },  /* level 13 */
-    { 17, 18, 16,  5,  4, ZSTD_HC_btlazy2 },  /* level 14 */
-    { 17, 18, 16,  6,  4, ZSTD_HC_btlazy2 },  /* level 15 */
-    { 17, 18, 16,  7,  4, ZSTD_HC_btlazy2 },  /* level 16 */
-    { 17, 18, 16,  8,  4, ZSTD_HC_btlazy2 },  /* level 17 */
-    { 17, 18, 16,  9,  4, ZSTD_HC_btlazy2 },  /* level 18 */
-    { 17, 18, 16, 10,  4, ZSTD_HC_btlazy2 },  /* level 19 */
-    { 17, 18, 18, 12,  4, ZSTD_HC_btlazy2 },  /* level 20 */
-},
-{   /* for > 128 KB */
-    /* W,  C,  H,  S,  L, strat */
-    { 18, 12, 12,  1,  4, ZSTD_HC_fast    },  /* level  0 - never used */
-    { 18, 14, 14,  1,  7, ZSTD_HC_fast    },  /* level  1 - in fact redirected towards zstd fast */
-    { 19, 15, 16,  1,  6, ZSTD_HC_fast    },  /* level  2 */
-    { 20, 18, 20,  1,  6, ZSTD_HC_fast    },  /* level  3 */
-    { 21, 19, 21,  1,  6, ZSTD_HC_fast    },  /* level  4 */
-    { 20, 13, 18,  5,  5, ZSTD_HC_greedy  },  /* level  5 */
-    { 20, 17, 19,  3,  5, ZSTD_HC_greedy  },  /* level  6 */
-    { 21, 17, 20,  3,  5, ZSTD_HC_lazy    },  /* level  7 */
-    { 21, 19, 20,  3,  5, ZSTD_HC_lazy    },  /* level  8 */
-    { 21, 20, 20,  3,  5, ZSTD_HC_lazy2   },  /* level  9 */
-    { 21, 19, 20,  4,  5, ZSTD_HC_lazy2   },  /* level 10 */
-    { 22, 20, 22,  4,  5, ZSTD_HC_lazy2   },  /* level 11 */
-    { 22, 20, 22,  5,  5, ZSTD_HC_lazy2   },  /* level 12 */
-    { 22, 21, 22,  5,  5, ZSTD_HC_lazy2   },  /* level 13 */
-    { 22, 22, 23,  5,  5, ZSTD_HC_lazy2   },  /* level 14 */
-    { 23, 23, 23,  5,  5, ZSTD_HC_lazy2   },  /* level 15 */
-    { 23, 21, 22,  5,  5, ZSTD_HC_btlazy2 },  /* level 16 */
-    { 23, 24, 23,  4,  5, ZSTD_HC_btlazy2 },  /* level 17 */
-    { 25, 24, 23,  5,  5, ZSTD_HC_btlazy2 },  /* level 18 */
-    { 25, 26, 23,  5,  5, ZSTD_HC_btlazy2 },  /* level 19 */
-    { 26, 27, 24,  6,  5, ZSTD_HC_btlazy2 },  /* level 20 */
-}
-};
-
-
-#if defined (__cplusplus)
-}
-#endif

programs/Makefile

@@ -30,12 +30,12 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-VERSION?= 0.3.6
+VERSION?= 0.4.0
 
 DESTDIR?=
 PREFIX ?= /usr/local
 CPPFLAGS= -I../lib -I../lib/legacy -I./legacy -DZSTD_VERSION=\"$(VERSION)\" -DZSTD_LEGACY_SUPPORT=1
-CFLAGS ?= -O3  # -falign-loops=32   # not always positive
+CFLAGS ?= -O3  # -falign-loops=32   # not always beneficial
 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
@@ -53,42 +53,55 @@ EXT =
 VOID = /dev/null
 endif
 
+ZBUFFTEST = -T2mn
+
+.PHONY: default all clean install uninstall test test32 test-all
 
 default: zstd
 
-all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
+all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
 
-zstd: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
       xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
       xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
-fullbench  : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+fullbench  : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
       datagen.c fullbench.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fullbench32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+fullbench32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
       datagen.c fullbench.c
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
-fuzzer  : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+fuzzer  : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
       datagen.c xxhash.c fuzzer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fuzzer32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+fuzzer32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
       datagen.c xxhash.c fuzzer.c
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
-paramgrill : $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+zbufftest  : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+      $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
+      datagen.c xxhash.c zbufftest.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+      $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
+      datagen.c xxhash.c zbufftest.c
+	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
+paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
              $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
              datagen.c xxhash.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
@@ -97,10 +110,10 @@ datagen : datagen.c datagencli.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 clean:
-	@rm -f core *.o tmp \
+	@rm -f core *.o tmp* \
         zstd$(EXT) zstd32$(EXT) \
         fullbench$(EXT) fullbench32$(EXT) \
-        fuzzer$(EXT) fuzzer32$(EXT) \
+        fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
 	datagen$(EXT) paramgrill$(EXT)
 	@echo Cleaning completed
 
@@ -130,9 +143,9 @@ uninstall:
 	[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
 	@echo zstd programs successfully uninstalled
 
-test: test-zstd test-fullbench test-fuzzer
+test: test-zstd test-fullbench test-fuzzer test-zbuff
 
-test32: test-zstd32 test-fullbench32 test-fuzzer32
+test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
 
 test-all: test test32 valgrindTest
 
@@ -153,11 +166,61 @@ test-zstd: zstd datagen
 	echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
 	echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
 	@echo "**** zstd round-trip tests **** "
-	./datagen          | ./zstd -v    | ./zstd -d > $(VOID)
+	@./datagen             | md5sum > tmp1
-	./datagen          | ./zstd -6 -v | ./zstd -d > $(VOID)
+	./datagen              | ./zstd -v    | ./zstd -d  | md5sum > tmp2
-	./datagen -g256MB  | ./zstd -v    | ./zstd -d > $(VOID)
+	@diff tmp1 tmp2   
-	./datagen -g256MB  | ./zstd -3 -v | ./zstd -d > $(VOID)
+	./datagen              | ./zstd -6 -v | ./zstd -d  | md5sum > tmp2
-	./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)
+	@diff tmp1 tmp2   
+	@./datagen -g257MB     | md5sum > tmp1
+	./datagen -g257MB      | ./zstd -v    | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g257MB      | ./zstd -v2   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g257MB      | ./zstd -v3   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	@./datagen -g129MB -P60| md5sum > tmp1
+	./datagen -g129MB -P60 | ./zstd -v4   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g129MB -P60 | ./zstd -v5   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g129MB -P60 | ./zstd -v6   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	@./datagen -g65MB -P70 | md5sum > tmp1
+	./datagen -g65MB -P70  | ./zstd -v7   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g65MB -P70  | ./zstd -v8   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g65MB -P70  | ./zstd -v9   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	@./datagen -g33MB -P75 | md5sum > tmp1
+	./datagen -g33MB -P75  | ./zstd -v10  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g33MB -P75  | ./zstd -v11  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g33MB -P75  | ./zstd -v12  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	@./datagen -g17MB -P80 | md5sum > tmp1
+	./datagen -g17MB -P80  | ./zstd -v13  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g17MB -P80  | ./zstd -v14  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g17MB -P80  | ./zstd -v15  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g17MB -P80  | ./zstd -v16  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g17MB -P80  | ./zstd -v17  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	@./datagen -g49MB -P93 | md5sum > tmp1
+	./datagen -g49MB -P93  | ./zstd -v18  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g49MB -P93  | ./zstd -v19  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	@./datagen -g97MB -P99 | md5sum > tmp1
+	./datagen -g97MB -P99  | ./zstd -v20  | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
+	./datagen -g6GB -P99   | md5sum > tmp1
+	./datagen -g6GB -P99   | ./zstd -vq   | ./zstd -d  | md5sum > tmp2
+	@diff tmp1 tmp2   
 
 test-zstd32: zstd32 datagen
 	./datagen          | ./zstd32 -v  | ./zstd32 -d > $(VOID)
@@ -178,17 +241,25 @@ test-fuzzer: fuzzer
 test-fuzzer32: fuzzer32
 	./fuzzer32
 
-valgrindTest: zstd datagen fuzzer fullbench
+test-zbuff: zbufftest
+	./zbufftest $(ZBUFFTEST)
+
+test-zbuff32: zbufftest32
+	./zbufftest32 $(ZBUFFTEST)
+
+valgrindTest: zstd datagen fuzzer fullbench zbufftest
 	@echo "\n ---- valgrind tests : memory analyzer ----"
 	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
 	./datagen -g16KB > tmp
 	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
 	./datagen -g2930KB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./zstd -4 -vf tmp $(VOID)
+	valgrind --leak-check=yes --error-exitcode=1 ./zstd -5 -vf tmp tmp2
+	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vdf tmp2 $(VOID)
 	./datagen -g64MB > tmp
 	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
 	@rm tmp
 	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
 	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
+	valgrind --leak-check=yes --error-exitcode=1 ./zbufftest -T1mn
 
 endif

programs/bench.c

@@ -61,7 +61,6 @@
 
 #include "mem.h"
 #include "zstd.h"
-#include "zstdhc.h"
 #include "xxhash.h"
 
 
@@ -231,12 +230,6 @@ typedef struct
 
 typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
 
-static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
-{
-    (void)compressionLevel;
-    return ZSTD_compress(dst, maxDstSize, src, srcSize);
-}
-
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 
 static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
@@ -247,7 +240,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
     const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* const resultBuffer = malloc(srcSize);
-    const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
+    const compressor_t compressor = ZSTD_compress;
     U64 crcOrig;
 
     /* init */
@@ -413,7 +406,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t)(requiredMem - step);
 }
 
-static int BMK_benchOneFile(char* inFileName, int cLevel)
+static int BMK_benchOneFile(const char* inFileName, int cLevel)
 {
     FILE*  inFile;
     U64    inFileSize;
@@ -513,7 +506,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility)
 }
 
 
-int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
 {
     double compressibility = (double)g_compressibilityDefault / 100;
 

programs/bench.h

@@ -26,7 +26,7 @@
 
 
 /* Main function */
-int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
 
 /* Set Parameters */
 void BMK_SetNbIterations(int nbLoops);

programs/datagen.c

@@ -81,7 +81,6 @@ typedef BYTE litDistribTable[LTSIZE];
 
 
 
-
 /*********************************************************
 *  Local Functions
 *********************************************************/
@@ -130,7 +129,7 @@ static BYTE RDG_genChar(U32* seed, const litDistribTable lt)
 }
 
 
-#define RDG_RAND15BITS  ((RDG_rand(seed) >> 3) & 32767)
+#define RDG_RAND15BITS  ((RDG_rand(seed) >> 3) & 0x7FFF)
 #define RDG_RANDLENGTH  ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
 void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr)
 {
@@ -138,6 +137,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
     const U32 matchProba32 = (U32)(32768 * matchProba);
     size_t pos = prefixSize;
     U32* seed = seedPtr;
+    U32 prevOffset = 1;
 
     /* special case : sparse content */
     while (matchProba >= 1.0)
@@ -170,11 +170,13 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
             size_t d;
             int length = RDG_RANDLENGTH + 4;
             U32 offset = RDG_RAND15BITS + 1;
+            U32 repeatOffset = (RDG_rand(seed) & 15) == 2;
+            if (repeatOffset) offset = prevOffset;
             if (offset > pos) offset = (U32)pos;
             match = pos - offset;
             d = pos + length;
             if (d > buffSize) d = buffSize;
-            while (pos < d) buffPtr[pos++] = buffPtr[match++];
+            while (pos < d) buffPtr[pos++] = buffPtr[match++];   /* correctly manages overlaps */
         }
         else
         {

programs/fileio.c

@@ -49,8 +49,6 @@
 #  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
 #endif
 
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
 #define _FILE_OFFSET_BITS 64   /* Large file support on 32-bits unix */
 #define _POSIX_SOURCE 1        /* enable fileno() within <stdio.h> on unix */
 
@@ -67,11 +65,11 @@
 #include <sys/stat.h>   /* stat64 */
 #include "mem.h"
 #include "fileio.h"
-#include "zstd_static.h"
+#include "zstd_static.h"   /* ZSTD_magicNumber */
-#include "zstdhc_static.h"
+#include "zstd_buffered_static.h"
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-#  include "zstd_legacy.h"  /* legacy */
+#  include "zstd_legacy.h"    /* legacy */
 #  include "fileio_legacy.h"  /* legacy */
 #endif
 
@@ -83,7 +81,7 @@
 #  include <fcntl.h>    /* _O_BINARY */
 #  include <io.h>       /* _setmode, _isatty */
 #  ifdef __MINGW32__
-   /* int _fileno(FILE *stream);   // seems no longer useful // MINGW somehow forgets to include this windows declaration into <stdio.h> */
+   // int _fileno(FILE *stream);   /* seems no longer useful /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
 #  endif
 #  define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
@@ -115,8 +113,8 @@
 #define BIT6  0x40
 #define BIT7  0x80
 
-//static const unsigned FIO_maxBlockSizeID = 0xB;   /* => 2MB block */
+#define BLOCKSIZE      (128 KB)
-static const unsigned FIO_blockHeaderSize = 3;
+#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
 
 #define FIO_FRAMEHEADERSIZE 5        /* as a define, because needed to allocated table on stack */
 #define FSE_CHECKSUM_SEED        0
@@ -237,102 +235,30 @@ static U64 FIO_getFileSize(const char* infilename)
 }
 
 
-typedef void* (*FIO_createC) (void);
-static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
-static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
-
-typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
-static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
-{
-    (void)cLevel; (void)srcSizeHint;
-    return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
-}
-static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
-{
-    return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
-}
-
-typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-static size_t local_ZSTD_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
-    return ZSTD_compressContinue((ZSTD_CCtx*)ctx, dst, maxDstSize, src, srcSize);
-}
-static size_t local_ZSTD_HC_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
-    return ZSTD_HC_compressContinue((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, src, srcSize);
-}
-
-typedef size_t (*FIO_endC) (void* ctx, void* dst, size_t maxDstSize);
-static size_t local_ZSTD_compressEnd (void* ctx, void* dst, size_t maxDstSize)
-{
-    return ZSTD_compressEnd((ZSTD_CCtx*)ctx, dst, maxDstSize);
-}
-static size_t local_ZSTD_HC_compressEnd (void* ctx, void* dst, size_t maxDstSize)
-{
-    return ZSTD_HC_compressEnd((ZSTD_HC_CCtx*)ctx, dst, maxDstSize);
-}
-
-typedef void (*FIO_freeC) (void* ctx);
-static void local_ZSTD_freeCCtx(void* ctx) { ZSTD_freeCCtx((ZSTD_CCtx*)ctx); }
-static void local_ZSTD_HC_freeCCtx(void* ctx) { ZSTD_HC_freeCCtx((ZSTD_HC_CCtx*)ctx); }
-
-
 unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
 {
     U64 filesize = 0;
     U64 compressedfilesize = 0;
     BYTE* inBuff;
-    BYTE* inSlot;
-    BYTE* inEnd;
     BYTE* outBuff;
-    size_t blockSize = 128 KB;
+    size_t inBuffSize = ZBUFF_recommendedCInSize();
-    size_t inBuffSize = 4 * blockSize;
+    size_t outBuffSize = ZBUFF_recommendedCOutSize();
-    size_t outBuffSize = ZSTD_compressBound(blockSize);
     FILE* finput;
     FILE* foutput;
-    size_t sizeCheck, cSize;
+    size_t sizeCheck, errorCode;
-    void* ctx;
+    ZBUFF_CCtx* ctx;
-    FIO_createC createC=NULL;
-    FIO_initC initC=NULL;
-    FIO_continueC continueC = NULL;
-    FIO_endC endC = NULL;
-    FIO_freeC freeC = NULL;
-
-    /* Init */
-    if (cLevel <= 1)
-    {
-        createC = local_ZSTD_createCCtx;
-        initC = local_ZSTD_compressBegin;
-        continueC = local_ZSTD_compressContinue;
-        endC = local_ZSTD_compressEnd;
-        freeC = local_ZSTD_freeCCtx;
-    }
-    else
-    {
-        createC = local_ZSTD_HC_createCCtx;
-        initC = local_ZSTD_HC_compressBegin;
-        continueC = local_ZSTD_HC_compressContinue;
-        endC = local_ZSTD_HC_compressEnd;
-        freeC = local_ZSTD_HC_freeCCtx;
-    }
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    filesize = FIO_getFileSize(input_filename);
 
     /* Allocate Memory */
-    ctx = createC();
+    ctx = ZBUFF_createCCtx();
     inBuff  = (BYTE*)malloc(inBuffSize);
     outBuff = (BYTE*)malloc(outBuffSize);
     if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
-    inSlot = inBuff;
-    inEnd = inBuff + inBuffSize;
 
-    /* Write Frame Header */
+    /* init */
-    cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
+    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
+    filesize = FIO_getFileSize(input_filename);
-
+    errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize));
-    sizeCheck = fwrite(outBuff, 1, cSize, foutput);
+    if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression");
-    if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
-    compressedfilesize += cSize;
     filesize = 0;
 
     /* Main compression loop */
@@ -341,33 +267,41 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
         size_t inSize;
 
         /* Fill input Buffer */
-        if (inSlot + blockSize > inEnd) inSlot = inBuff;
+        inSize = fread(inBuff, (size_t)1, inBuffSize, finput);
-        inSize = fread(inSlot, (size_t)1, blockSize, finput);
         if (inSize==0) break;
         filesize += inSize;
-        DISPLAYUPDATE(2, "\rRead : %u MB   ", (U32)(filesize>>20));
+        DISPLAYUPDATE(2, "\rRead : %u MB  ", (U32)(filesize>>20));
 
-        /* Compress Block */
+        {
-        cSize = continueC(ctx, outBuff, outBuffSize, inSlot, inSize);
+            /* Compress (buffered streaming ensures appropriate formatting) */
-        if (ZSTD_isError(cSize))
+            size_t usedInSize = inSize;
-            EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
+            size_t cSize = outBuffSize;
+            size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize);
+            if (ZBUFF_isError(result))
+                EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
+            if (inSize != usedInSize)
+                /* inBuff should be entirely consumed since buffer sizes are recommended ones */
+                EXM_THROW(24, "Compression error : input block not fully consumed");
 
-        /* Write cBlock */
+            /* Write cBlock */
-        sizeCheck = fwrite(outBuff, 1, cSize, foutput);
+            sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-        if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
+            if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
-        compressedfilesize += cSize;
+            compressedfilesize += cSize;
-        inSlot += inSize;
+        }
 
         DISPLAYUPDATE(2, "\rRead : %u MB  ==> %.2f%%   ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
     }
 
     /* End of Frame */
-    cSize = endC(ctx, outBuff, outBuffSize);
+    {
-    if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
+        size_t cSize = outBuffSize;
+        size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize);
+        if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
 
-    sizeCheck = fwrite(outBuff, 1, cSize, foutput);
+        sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-    if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
+        if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
-    compressedfilesize += cSize;
+        compressedfilesize += cSize;
+    }
 
     /* Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
@@ -377,7 +311,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     /* clean */
     free(inBuff);
     free(outBuff);
-    freeC(ctx);
+    ZBUFF_freeCCtx(ctx);
     fclose(finput);
     if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
 
@@ -386,124 +320,87 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 
 
 unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
-                                       BYTE* inBuff, size_t inBuffSize,
+                                       BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
                                        BYTE* outBuff, size_t outBuffSize,
-                                       ZSTD_DCtx* dctx)
+                                       ZBUFF_DCtx* dctx)
 {
-    BYTE* op = outBuff;
+    U64    frameSize = 0;
-    BYTE* const oend = outBuff + outBuffSize;
+    size_t readSize=alreadyLoaded;
-    U64   filesize = 0;
-    size_t toRead;
-    size_t sizeCheck;
-
 
     /* Main decompression Loop */
-    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+    ZBUFF_decompressInit(dctx);
-    while (toRead)
+    while (1)
     {
-        size_t readSize, decodedSize;
+        /* Decode */
+        size_t sizeCheck;
+        size_t inSize=readSize, decodedSize=outBuffSize;
+        size_t inStart=0;
+        size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
+        if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
+        if (toRead==0) break;   /* end of Frame */
+        readSize -= inSize;
+        inStart += inSize;
+
+        /* Write block */
+        sizeCheck = fwrite(outBuff, 1, decodedSize, foutput);
+        if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
+        frameSize += decodedSize;
+        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
+
+        if (readSize) continue;   /* still some data left within inBuff */
 
         /* Fill input buffer */
-        if (toRead > inBuffSize)
+        if (toRead > inBuffSize) EXM_THROW(34, "too large block");
-            EXM_THROW(34, "too large block");
         readSize = fread(inBuff, 1, toRead, finput);
-        if (readSize != toRead)
+        if (readSize != toRead) EXM_THROW(35, "Read error");
-            EXM_THROW(35, "Read error");
-
-        /* Decode block */
-        decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
-        if (ZSTD_isError(decodedSize)) EXM_THROW(36, "Decoding error : input corrupted");
-
-        if (decodedSize)   /* not a header */
-        {
-            /* Write block */
-            sizeCheck = fwrite(op, 1, decodedSize, foutput);
-            if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
-            filesize += decodedSize;
-            op += decodedSize;
-            if (op==oend) op = outBuff;
-            DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
-        }
-
-        /* prepare for next Block */
-        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     }
 
-    return filesize;
+    return frameSize;
 }
 
 
-#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3)
 unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
 {
     FILE* finput, *foutput;
     BYTE* inBuff=NULL;
-    size_t inBuffSize = 0;
+    size_t inBuffSize = ZBUFF_recommendedDInSize();
     BYTE* outBuff=NULL;
-    size_t outBuffSize = 0;
+    size_t outBuffSize = ZBUFF_recommendedDOutSize();
-    U32   blockSize = 128 KB;
-    U32   wNbBlocks = 4;
     U64   filesize = 0;
-    BYTE* header[MAXHEADERSIZE];
     size_t toRead;
     size_t sizeCheck;
 
 
     /* Init */
-    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    ZBUFF_DCtx* dctx = ZBUFF_createDCtx();
     FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
 
+    /* Allocate Memory (if needed) */
+    inBuff  = (BYTE*)malloc(inBuffSize);
+    outBuff  = (BYTE*)malloc(outBuffSize);
+    if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+
     /* for each frame */
     for ( ; ; )
     {
-        /* check magic number -> version */
         U32 magicNumber;
-        toRead = sizeof(ZSTD_magicNumber);;
+        toRead = 0;
-        sizeCheck = fread(header, (size_t)1, toRead, finput);
+
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+        /* check magic number -> version */
+        toRead = 4;
+        sizeCheck = fread(inBuff, (size_t)1, toRead, finput);
         if (sizeCheck==0) break;   /* no more input */
         if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-
+        magicNumber = MEM_readLE32(inBuff);
-        magicNumber = MEM_readLE32(header);
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
         if (ZSTD_isLegacy(magicNumber))
         {
             filesize += FIO_decompressLegacyFrame(foutput, finput, magicNumber);
             continue;
         }
 #endif   /* ZSTD_LEGACY_SUPPORT */
-        if (magicNumber != ZSTD_magicNumber) EXM_THROW(32, "Error : unknown frame prefix");
 
-        /* prepare frame decompression, by completing header */
+        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx);
-        ZSTD_resetDCtx(dctx);
-        toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber);
-        if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
-        sizeCheck = fread(&header[sizeof(ZSTD_magicNumber)], 1, toRead, finput);
-        if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-        sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead);   // Decode frame header
-        if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
-        /* Here later : blockSize determination */
-
-        /* Allocate Memory (if needed) */
-        {
-            size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
-            size_t newOutBuffSize = wNbBlocks * blockSize;
-            if (newInBuffSize > inBuffSize)
-            {
-                free(inBuff);
-                inBuffSize = newInBuffSize;
-                inBuff  = (BYTE*)malloc(inBuffSize);
-            }
-            if (newOutBuffSize > outBuffSize)
-            {
-                free(outBuff);
-                outBuffSize = newOutBuffSize;
-                outBuff  = (BYTE*)malloc(outBuffSize);
-            }
-        }
-        if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
-        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
     }
 
     DISPLAYLEVEL(2, "\r%79s\r", "");
@@ -512,7 +409,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
     /* clean */
     free(inBuff);
     free(outBuff);
-    ZSTD_freeDCtx(dctx);
+    ZBUFF_freeDCtx(dctx);
     fclose(finput);
     if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
 
@@ -520,74 +417,3 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
 }
 
 
-#if 0
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
-{
-    FILE* finput, *foutput;
-    BYTE* inBuff=NULL;
-    size_t inBuffSize = 0;
-    BYTE* outBuff=NULL;
-    size_t outBuffSize = 0;
-    U32   blockSize = 128 KB;
-    U32   wNbBlocks = 4;
-    U64   filesize = 0;
-    BYTE* header[MAXHEADERSIZE];
-    ZSTD_Dctx* dctx;
-    size_t toRead;
-    size_t sizeCheck;
-
-
-    /* Init */
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    dctx = ZSTD_createDCtx();
-
-    /* for each frame */
-    for ( ; ; )
-    {
-        /* check header */
-        ZSTD_resetDCtx(dctx);
-        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-        if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
-        sizeCheck = fread(header, (size_t)1, toRead, finput);
-        if (sizeCheck==0) break;   /* no more input */
-        if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-        sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
-        if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
-        /* Here later : blockSize determination */
-
-        /* Allocate Memory (if needed) */
-        {
-            size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
-            size_t newOutBuffSize = wNbBlocks * blockSize;
-            if (newInBuffSize > inBuffSize)
-            {
-                free(inBuff);
-                inBuffSize = newInBuffSize;
-                inBuff  = (BYTE*)malloc(inBuffSize);
-            }
-            if (newOutBuffSize > outBuffSize)
-            {
-                free(outBuff);
-                outBuffSize = newOutBuffSize;
-                outBuff  = (BYTE*)malloc(outBuffSize);
-            }
-        }
-        if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
-        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
-    }
-
-    DISPLAYLEVEL(2, "\r%79s\r", "");
-    DISPLAYLEVEL(2, "Decoded %llu bytes   \n", (long long unsigned)filesize);
-
-    /* clean */
-    free(inBuff);
-    free(outBuff);
-    ZSTD_freeDCtx(dctx);
-    fclose(finput);
-    if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
-
-    return filesize;
-}
-#endif

programs/fullbench.c

@@ -61,7 +61,7 @@
 #endif
 
 #include "mem.h"
-#include "zstd.h"
+#include "zstd_static.h"
 #include "fse_static.h"
 #include "datagen.h"
 
@@ -209,7 +209,7 @@ typedef struct
 } blockProperties_t;
 
 static size_t g_cSize = 0;
-static U32 g_litCtx[40 * 1024];
+static ZSTD_DCtx* g_dctxPtr = NULL;
 
 extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
 extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
@@ -217,7 +217,7 @@ extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
 size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
 {
     (void)buff2;
-    return ZSTD_compress(dst, dstSize, src, srcSize);
+    return ZSTD_compress(dst, dstSize, src, srcSize, 1);
 }
 
 size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
@@ -226,11 +226,11 @@ size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void*
     return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
 }
 
-extern size_t ZSTD_decodeLiteralsBlock(void* ctx, const void* src, size_t srcSize);
+extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
 size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
 {
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
-    return ZSTD_decodeLiteralsBlock(g_litCtx, buff2, g_cSize);
+    return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_dctxPtr, buff2, g_cSize);
 }
 
 size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
@@ -243,31 +243,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
     return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
 }
 
-size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
-{
-    U32 i;
-    size_t total = 0;
-    BYTE* data = (BYTE*)buff2;
-
-    (void)dst; (void)dstSize; (void)src;
-    for (i=0; i < srcSize; i++)
-    {
-        U32 b = data[i];
-        total += b;
-        if (b==0) total = 0;   // 825
-        //if (!b) total = 0;     // 825
-        //total = b ? total : 0; // 622
-        //total &= -!b;          // 622
-        //total *= !!b;          // 465
-    }
-    return total;
-}
-
-size_t local_decodeLiteralsForward(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
-{
-    (void)src; (void)srcSize;
-    return FSE_decompress(dst, dstSize, buff2, g_cSize);
-}
 
 
 
@@ -300,12 +275,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
     case 32:
         benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
         break;
-    case 101:
-        benchFunction = local_conditionalNull; benchName = "conditionalNull";
-        break;
-    case 102:
-        benchFunction = local_decodeLiteralsForward; benchName = "ZSTD_decodeLiteralsForward";
-        break;
     default :
         return 0;
     }
@@ -314,6 +283,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
     dstBuffSize = ZSTD_compressBound(srcSize);
     dstBuff = (BYTE*)malloc(dstBuffSize);
     buff2 = (BYTE*)malloc(dstBuffSize);
+    g_dctxPtr = ZSTD_createDCtx();
     if ((!dstBuff) || (!buff2))
     {
         DISPLAY("\nError: not enough memory!\n");
@@ -325,22 +295,20 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
     switch(benchNb)
     {
     case 11:
-        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize);
+        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
         break;
     case 31:  /* ZSTD_decodeLiteralsBlock */
         {
             blockProperties_t bp;
-            g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
+            g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
-            ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp);   // Get first block type
+            ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp);  /* Get 1st block type */
             if (bp.blockType != bt_compressed)
             {
                 DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
-                free(dstBuff);
+                goto _cleanOut;
-                free(buff2);
-                return 0;
             }
-            memcpy(buff2, dstBuff+7, g_cSize-7);
+            memcpy(buff2, dstBuff+8, g_cSize-8);
-            srcSize = srcSize > 128 KB ? 128 KB : srcSize;   // relative to block
+            srcSize = srcSize > 128 KB ? 128 KB : srcSize;    /* speed relative to block */
             break;
         }
     case 32:   /* ZSTD_decodeSeqHeaders */
@@ -349,44 +317,26 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
             const BYTE* ip = dstBuff;
             const BYTE* iend;
             size_t blockSize;
-            ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
+            ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);   /* it would be better to use direct block compression here */
-            ip += 4;   // Jump magic Number
+            ip += 5;   /* Skip frame Header */
-            blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp);   // Get first block type
+            blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp);   /* Get 1st block type */
             if (bp.blockType != bt_compressed)
             {
                 DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
-                free(dstBuff);
+                goto _cleanOut;
-                free(buff2);
-                return 0;
             }
-            iend = ip + 3 + blockSize;   // Get end of first block
+            iend = ip + 3 + blockSize;   /* End of first block */
-            ip += 3;   // jump first block header
+            ip += 3;                     /* skip block header */
-            ip += ZSTD_decodeLiteralsBlock(g_litCtx, ip, iend-ip);  // jump literal sub block and its header
+            ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip);  /* skip literal segment */
             g_cSize = iend-ip;
-            memcpy(buff2, ip, g_cSize);   // copy rest of block (starting with SeqHeader)
+            memcpy(buff2, ip, g_cSize);   /* copy rest of block (it starts by SeqHeader) */
-            srcSize = srcSize > 128 KB ? 128 KB : srcSize;   // speed relative to block
+            srcSize = srcSize > 128 KB ? 128 KB : srcSize;   /* speed relative to block */
             break;
         }
 
     /* test functions */
+    /* by convention, test functions can be added > 100 */
 
-    case 101:   /* conditionalNull */
-        {
-            size_t i;
-            for (i=0; i<srcSize; i++)
-                buff2[i] = i & 15;
-            break;
-        }
-    case 102:   /* local_decodeLiteralsForward */
-        {
-            blockProperties_t bp;
-            ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
-            g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
-            memcpy(buff2, dstBuff+10, g_cSize);
-            //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);   // real speed
-            srcSize = srcSize > 128 KB ? 128 KB : srcSize;   // relative to block
-            break;
-        }
     default : ;
     }
 
@@ -418,8 +368,10 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
 
     DISPLAY("%2u- %-30.30s : %7.1f MB/s  (%9u)\n", benchNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode);
 
+_cleanOut:
     free(dstBuff);
     free(buff2);
+    ZSTD_freeDCtx(g_dctxPtr);
     return 0;
 }
 

programs/fuzzer.c

@@ -47,7 +47,6 @@
 #include <sys/timeb.h>   /* timeb */
 #include <string.h>      /* strcmp */
 #include "zstd_static.h"
-#include "zstdhc_static.h"
 #include "datagen.h"     /* RDG_genBuffer */
 #include "xxhash.h"      /* XXH64 */
 #include "mem.h"
@@ -64,7 +63,7 @@
 #define MB *(1U<<20)
 #define GB *(1U<<30)
 
-static const U32 nbTestsDefault = 32 KB;
+static const U32 nbTestsDefault = 30000;
 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
 static const U32 prime1 = 2654435761U;
@@ -90,6 +89,8 @@ static U32 g_time = 0;
 /*********************************************************
 *  Fuzzer functions
 *********************************************************/
+#define MAX(a,b) ((a)>(b)?(a):(b))
+
 static U32 FUZ_GetMilliStart(void)
 {
     struct timeb tb;
@@ -159,7 +160,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     /* Basic tests */
     DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
+    result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
     if (ZSTD_isError(result)) goto _output_error;
     cSize = result;
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
@@ -213,7 +214,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         sampleSize += 256 KB - 1;
         RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
         sampleSize += 96 KB;
-        cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize);
+        cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
         if (ZSTD_isError(cSize)) goto _output_error;
         result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
         if (ZSTD_isError(result)) goto _output_error;
@@ -265,11 +266,9 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     U32 testNb = 0;
     U32 coreSeed = seed, lseed = 0;
     ZSTD_CCtx* ctx;
-    ZSTD_HC_CCtx* hcctx;
 
     /* allocation */
     ctx = ZSTD_createCCtx();
-    hcctx = ZSTD_HC_createCCtx();
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
     cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
     cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
@@ -277,7 +276,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
     dstBuffer = (BYTE*)malloc (dstBufferSize);
     cBuffer   = (BYTE*)malloc (cBufferSize);
-    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx || !hcctx,
+    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx,
            "Not enough memory, fuzzer tests cancelled");
 
     /* Create initial samples */
@@ -300,6 +299,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
         U32 sampleSizeLog, buffNb, cLevelMod;
         U64 crcOrig, crcDest;
         int cLevel;
+        BYTE* sampleBuffer;
 
         /* init */
         DISPLAYUPDATE(2, "\r%6u/%6u   ", testNb, nbTests);
@@ -326,14 +326,18 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
         sampleSize = (size_t)1 << sampleSizeLog;
         sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
         sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
-        crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
 
-        /* HC compression test */
+        /* create sample buffer (to catch read error with valgrind & sanitizers)  */
-#define MAX(a,b) ((a)>(b)?(a):(b))
+        sampleBuffer = (BYTE*)malloc(sampleSize);
+        CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
+        memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
+        crcOrig = XXH64(sampleBuffer, sampleSize, 0);
+
+        /* compression test */
         cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2));   /* use high compression levels with small samples, for speed */
         cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
-        cSize = ZSTD_HC_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
+        cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
-        CHECK(ZSTD_isError(cSize), "ZSTD_HC_compressCCtx failed");
+        CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
 
         /* compression failure test : too small dest buffer */
         if (cSize > 3)
@@ -344,10 +348,10 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
             static const U32 endMark = 0x4DC2B1A9;
             U32 endCheck;
             memcpy(dstBuffer+tooSmallSize, &endMark, 4);
-            errorCode = ZSTD_HC_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
+            errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
-            CHECK(!ZSTD_isError(errorCode), "ZSTD_HC_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
+            CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
             memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
-            CHECK(endCheck != endMark, "ZSTD_HC_compressCCtx : dst buffer overflow");
+            CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
         }
 
         /* successfull decompression tests*/
@@ -355,7 +359,9 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
         dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
         CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
         crcDest = XXH64(dstBuffer, sampleSize, 0);
-        CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
+        CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
+
+        free(sampleBuffer);   /* no longer useful after this point */
 
         /* truncated src decompression test */
         {
@@ -434,7 +440,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
 
 _cleanup:
     ZSTD_freeCCtx(ctx);
-    ZSTD_HC_freeCCtx(hcctx);
     free(cNoiseBuffer[0]);
     free(cNoiseBuffer[1]);
     free(cNoiseBuffer[2]);

programs/paramgrill.c

@@ -43,6 +43,10 @@
 #  define BMK_LEGACY_TIMER 1
 #endif
 
+#if defined(_MSC_VER)
+#  define snprintf _snprintf    /* snprintf unsupported by Visual <= 2012 */
+#endif
+
 
 /**************************************
 *  Includes
@@ -62,8 +66,7 @@
 #endif
 
 #include "mem.h"
-#include "zstdhc_static.h"
+#include "zstd_static.h"
-#include "zstd.h"
 #include "datagen.h"
 #include "xxhash.h"
 
@@ -122,8 +125,7 @@ static U32 g_rand = 1;
 static U32 g_singleRun = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0];
+static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
-static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
 
 void BMK_SetNbIterations(int nbLoops)
 {
@@ -136,28 +138,6 @@ void BMK_SetNbIterations(int nbLoops)
 *  Private functions
 *********************************************************/
 
-static unsigned BMK_highbit(U32 val)
-{
-#   if defined(_MSC_VER)   /* Visual */
-    unsigned long r;
-    _BitScanReverse(&r, val);
-    return (unsigned)r;
-#   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
-    return 31 - __builtin_clz(val);
-#   else   /* Software version */
-    static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
-    U32 v = val;
-    int r;
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
-    return r;
-#   endif
-}
-
 #if defined(BMK_LEGACY_TIMER)
 
 static int BMK_GetMilliStart(void)
@@ -270,8 +250,8 @@ typedef struct
 
 static size_t BMK_benchParam(BMK_result_t* resultPtr,
                              const void* srcBuffer, size_t srcSize,
-                             ZSTD_HC_CCtx* ctx,
+                             ZSTD_CCtx* ctx,
-                             const ZSTD_HC_parameters params)
+                             const ZSTD_parameters params)
 {
     const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
     const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
@@ -284,7 +264,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
     U32 Hlog = params.hashLog;
     U32 Slog = params.searchLog;
     U32 Slength = params.searchLength;
-    ZSTD_HC_strategy strat = params.strategy;
+    ZSTD_strategy strat = params.strategy;
     char name[30] = { 0 };
     U64 crcOrig;
 
@@ -358,7 +338,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
             while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
             {
                 for (blockNb=0; blockNb<nbBlocks; blockNb++)
-                    blockTable[blockNb].cSize = ZSTD_HC_compress_advanced(ctx,
+                    blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
                                                     blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
                                                     blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
                                                     params);
@@ -429,13 +409,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
 }
 
 
-const char* g_stratName[] = { "ZSTD_HC_fast   ",
+const char* g_stratName[] = { "ZSTD_fast   ",
-                              "ZSTD_HC_greedy ",
+                              "ZSTD_greedy ",
-                              "ZSTD_HC_lazy   ",
+                              "ZSTD_lazy   ",
-                              "ZSTD_HC_lazy2  ",
+                              "ZSTD_lazy2  ",
-                              "ZSTD_HC_btlazy2" };
+                              "ZSTD_btlazy2" };
 
-static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
+static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
 {
     DISPLAY("\r%79s\r", "");
     fprintf(f,"    {%3u,%3u,%3u,%3u,%3u, %s },  ",
@@ -447,11 +427,11 @@ static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_pa
 }
 
 
-static U32 g_cSpeedTarget[ZSTD_HC_MAX_CLEVEL+1] = { 0 };
+static U32 g_cSpeedTarget[ZSTD_MAX_CLEVEL+1] = { 0 };
 
 typedef struct {
     BMK_result_t result;
-    ZSTD_HC_parameters params;
+    ZSTD_parameters params;
 } winnerInfo_t;
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -459,11 +439,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
     int cLevel;
 
     fprintf(f, "\n /* Selected configurations : */ \n");
-    fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
+    fprintf(f, "#define ZSTD_MAX_CLEVEL %2u \n", ZSTD_MAX_CLEVEL);
-    fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
+    fprintf(f, "static const ZSTD_parameters ZSTD_defaultParameters[ZSTD_MAX_CLEVEL+1] = {\n");
     fprintf(f, "    /* W,  C,  H,  S,  L, strat */ \n");
 
-    for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
+    for (cLevel=0; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
         BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
 }
 
@@ -477,9 +457,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
 }
 
 
-static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
+static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
               const void* srcBuffer, size_t srcSize,
-                    ZSTD_HC_CCtx* ctx)
+                    ZSTD_CCtx* ctx)
 {
     BMK_result_t testResult;
     int better = 0;
@@ -487,7 +467,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
 
     BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
 
-    for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
+    for (cLevel = 1; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
     {
         if (testResult.cSpeed < g_cSpeedTarget[cLevel])
             continue;   /* not fast enough for this level */
@@ -514,9 +494,9 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
             double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
 
             size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) +
-                                ((params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << params.contentLog));
+                                ((params.strategy==ZSTD_fast) ? 0 : 4 * (1 << params.contentLog));
             size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) +
-                                ((winners[cLevel].params.strategy==ZSTD_HC_fast) ? 0 :  4 * (1 << winners[cLevel].params.contentLog));
+                                ((winners[cLevel].params.strategy==ZSTD_fast) ? 0 :  4 * (1 << winners[cLevel].params.contentLog));
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
 
@@ -581,10 +561,10 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
 
 
 /* nullified useless params, to ensure count stats */
-static ZSTD_HC_parameters* sanitizeParams(ZSTD_HC_parameters params)
+static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
 {
     g_params = params;
-    if (params.strategy == ZSTD_HC_fast)
+    if (params.strategy == ZSTD_fast)
     {
         g_params.contentLog = 0;
         g_params.searchLog = 0;
@@ -604,16 +584,16 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0};   /* init to zero */
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 
 static void playAround(FILE* f, winnerInfo_t* winners,
-                       ZSTD_HC_parameters params,
+                       ZSTD_parameters params,
                        const void* srcBuffer, size_t srcSize,
-                       ZSTD_HC_CCtx* ctx)
+                       ZSTD_CCtx* ctx)
 {
     int nbVariations = 0;
     const int startTime = BMK_GetMilliStart();
 
     while (BMK_GetMilliSpan(startTime) < g_maxVariationTime)
     {
-        ZSTD_HC_parameters p = params;
+        ZSTD_parameters p = params;
         U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
         if (nbVariations++ > g_maxNbVariations) break;
 
@@ -643,16 +623,16 @@ static void playAround(FILE* f, winnerInfo_t* winners,
             case 9:
                 p.searchLength--; break;
             case 10:
-                p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)+1); break;
+                p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
             case 11:
-                p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)-1); break;
+                p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
             }
         }
 
         /* validate new conf */
         {
-            ZSTD_HC_parameters saved = p;
+            ZSTD_parameters saved = p;
-            ZSTD_HC_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
+            ZSTD_validateParams(&p);
             if (memcmp(&p, &saved, sizeof(p))) continue;  /* p was invalid */
         }
 
@@ -675,19 +655,19 @@ static void playAround(FILE* f, winnerInfo_t* winners,
 static void BMK_selectRandomStart(
                        FILE* f, winnerInfo_t* winners,
                        const void* srcBuffer, size_t srcSize,
-                       ZSTD_HC_CCtx* ctx)
+                       ZSTD_CCtx* ctx)
 {
-    U32 id = (FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1));
+    U32 id = (FUZ_rand(&g_rand) % (ZSTD_MAX_CLEVEL+1));
     if ((id==0) || (winners[id].params.windowLog==0))
     {
         /* totally random entry */
-        ZSTD_HC_parameters p;
+        ZSTD_parameters p;
-        p.contentLog = FUZ_rand(&g_rand) % (ZSTD_HC_CONTENTLOG_MAX+1 - ZSTD_HC_CONTENTLOG_MIN) + ZSTD_HC_CONTENTLOG_MIN;
+        p.contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
-        p.hashLog    = FUZ_rand(&g_rand) % (ZSTD_HC_HASHLOG_MAX+1 - ZSTD_HC_HASHLOG_MIN) + ZSTD_HC_HASHLOG_MIN;
+        p.hashLog    = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
-        p.searchLog  = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
+        p.searchLog  = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
-        p.windowLog  = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
+        p.windowLog  = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
-        p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN;
+        p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
-        p.strategy   = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % (ZSTD_HC_btlazy2+1));
+        p.strategy   = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btlazy2+1));
         playAround(f, winners, p, srcBuffer, srcSize, ctx);
     }
     else
@@ -697,19 +677,19 @@ static void BMK_selectRandomStart(
 
 static void BMK_benchMem(void* srcBuffer, size_t srcSize)
 {
-    ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx();
+    ZSTD_CCtx* ctx = ZSTD_createCCtx();
-    ZSTD_HC_parameters params;
+    ZSTD_parameters params;
-    winnerInfo_t winners[ZSTD_HC_MAX_CLEVEL+1];
+    winnerInfo_t winners[ZSTD_MAX_CLEVEL+1];
     int i;
     const char* rfName = "grillResults.txt";
     FILE* f;
     const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
-    const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1;
 
     if (g_singleRun)
     {
         BMK_result_t testResult;
-        ZSTD_HC_validateParams(&g_params, blockSize);
+        g_params.srcSize = blockSize;
+        ZSTD_validateParams(&g_params);
         BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
         DISPLAY("\n");
         return;
@@ -731,28 +711,23 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
         params.contentLog = 1;
         params.searchLog = 1;
         params.searchLength = 7;
-        params.strategy = ZSTD_HC_fast;
+        params.strategy = ZSTD_fast;
-        ZSTD_HC_validateParams(&params, blockSize);
+        params.srcSize = blockSize;
+        ZSTD_validateParams(&params);
         BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
-        g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
+        g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
     }
 
     /* establish speed objectives (relative to level 1) */
-    for (i=2; i<=ZSTD_HC_MAX_CLEVEL; i++)
+    for (i=2; i<=ZSTD_MAX_CLEVEL; i++)
         g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5;
 
     /* populate initial solution */
     {
-        const int tableID = (blockSize > 128 KB);
+        const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
-        const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
-        g_seedParams = ZSTD_HC_defaultParameters[tableID];
         for (i=1; i<=maxSeeds; i++)
         {
-            const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2);
+            params = ZSTD_getParams(i, blockSize);
-            params = g_seedParams[i];
-            params.windowLog = MIN(srcLog, params.windowLog);
-            params.contentLog = MIN(params.windowLog+btPlus, params.contentLog);
-            params.searchLog = MIN(params.contentLog, params.searchLog);
             BMK_seed(winners, params, srcBuffer, srcSize, ctx);
         }
     }
@@ -775,7 +750,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
 
     /* clean up*/
     fclose(f);
-    ZSTD_HC_freeCCtx(ctx);
+    ZSTD_freeCCtx(ctx);
 }
 
 
@@ -874,6 +849,7 @@ int usage(char* exename)
     DISPLAY( "Usage :\n");
     DISPLAY( "      %s [arg] file\n", exename);
     DISPLAY( "Arguments :\n");
+    DISPLAY( " file : path to the file used as reference (if none, generates a compressible sample)\n");
     DISPLAY( " -H/-h  : Help (this text + advanced options)\n");
     return 0;
 }
@@ -882,7 +858,8 @@ int usage_advanced(void)
 {
     DISPLAY( "\nAdvanced options :\n");
     DISPLAY( " -i#    : iteration loops [1-9](default : %i)\n", NBLOOPS);
-    DISPLAY( " -P#    : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
+    DISPLAY( " -B#    : cut input into blocks of size # (default : single block)\n");
+    DISPLAY( " -P#    : generated sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
     return 0;
 }
 
@@ -958,7 +935,7 @@ int main(int argc, char** argv)
                 case 'S':
                     g_singleRun = 1;
                     argument++;
-                    g_params = g_seedParams[2];
+                    g_params = ZSTD_getParams(2, g_blockSize);
                     for ( ; ; )
                     {
                         switch(*argument)
@@ -994,12 +971,12 @@ int main(int argc, char** argv)
                                 g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
                             continue;
                         case 't':  /* strategy */
-                            g_params.strategy = (ZSTD_HC_strategy)0;
+                            g_params.strategy = (ZSTD_strategy)0;
                             argument++;
                             while ((*argument>= '0') && (*argument<='9'))
                             {
-                                g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy *10);
+                                g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10);
-                                g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy + *argument++ - '0');
+                                g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0');
                             }
                             continue;
                         case 'L':
@@ -1008,9 +985,7 @@ int main(int argc, char** argv)
                                 argument++;
                                 while ((*argument>= '0') && (*argument<='9'))
                                     cLevel *= 10, cLevel += *argument++ - '0';
-                                if (cLevel < 1) cLevel = 1;
+                                g_params = ZSTD_getParams(cLevel, g_blockSize);
-                                if (cLevel > ZSTD_HC_MAX_CLEVEL) cLevel = ZSTD_HC_MAX_CLEVEL;
-                                g_params = g_seedParams[cLevel];
                                 continue;
                             }
                         default : ;

programs/zbufftest.c

@@ -0,0 +1,587 @@
+/*
+    Fuzzer test tool for zstd_buffered
+    Copyright (C) Yann Collet 2105
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - ZSTD source repository : https://github.com/Cyan4973/zstd
+    - ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/**************************************
+*  Compiler specific
+**************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  define _CRT_SECURE_NO_WARNINGS     /* fgets */
+#  pragma warning(disable : 4127)     /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4146)     /* disable: C4146: minus unsigned expression */
+#endif
+
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#ifdef __GNUC__
+#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
+#  pragma GCC diagnostic ignored "-Wmissing-field-initializers"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
+#endif
+
+
+/**************************************
+*  Includes
+**************************************/
+#include <stdlib.h>      /* free */
+#include <stdio.h>       /* fgets, sscanf */
+#include <sys/timeb.h>   /* timeb */
+#include <string.h>      /* strcmp */
+#include "mem.h"
+#include "zstd_buffered.h"
+#include "zstd.h"        /* ZSTD_compressBound() */
+#include "datagen.h"     /* RDG_genBuffer */
+#include "xxhash.h"      /* XXH64 */
+
+
+/**************************************
+ Constants
+**************************************/
+#ifndef ZSTD_VERSION
+#  define ZSTD_VERSION ""
+#endif
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+static const U32 nbTestsDefault = 10000;
+#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
+#define FUZ_COMPRESSIBILITY_DEFAULT 50
+static const U32 prime1 = 2654435761U;
+static const U32 prime2 = 2246822519U;
+
+
+
+/**************************************
+*  Display Macros
+**************************************/
+#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2;
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stdout); } }
+static const U32 g_refreshRate = 150;
+static U32 g_displayTime = 0;
+
+static U32 g_testTime = 0;
+
+
+/*********************************************************
+*  Fuzzer functions
+*********************************************************/
+#define MAX(a,b) ((a)>(b)?(a):(b))
+
+static U32 FUZ_GetMilliStart(void)
+{
+    struct timeb tb;
+    U32 nCount;
+    ftime( &tb );
+    nCount = (U32) (((tb.time & 0xFFFFF) * 1000) +  tb.millitm);
+    return nCount;
+}
+
+
+static U32 FUZ_GetMilliSpan(U32 nTimeStart)
+{
+    U32 nCurrent = FUZ_GetMilliStart();
+    U32 nSpan = nCurrent - nTimeStart;
+    if (nTimeStart > nCurrent)
+        nSpan += 0x100000 * 1000;
+    return nSpan;
+}
+
+
+#  define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+unsigned int FUZ_rand(unsigned int* src)
+{
+    U32 rand32 = *src;
+    rand32 *= prime1;
+    rand32 += prime2;
+    rand32  = FUZ_rotl32(rand32, 13);
+    *src = rand32;
+    return rand32 >> 5;
+}
+
+
+/*
+static unsigned FUZ_highbit32(U32 v32)
+{
+    unsigned nbBits = 0;
+    if (v32==0) return 0;
+    for ( ; v32 ; v32>>=1) nbBits++;
+    return nbBits;
+}
+*/
+
+static int basicUnitTests(U32 seed, double compressibility)
+{
+    int testResult = 0;
+    void* CNBuffer;
+    size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
+    void* compressedBuffer;
+    size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
+    void* decodedBuffer;
+    size_t decodedBufferSize = CNBufferSize;
+    U32 randState = seed;
+    size_t result, cSize, readSize, genSize;
+    U32 testNb=0;
+    ZBUFF_CCtx* zc = ZBUFF_createCCtx();
+    ZBUFF_DCtx* zd = ZBUFF_createDCtx();
+
+    /* Create compressible test buffer */
+    CNBuffer = malloc(CNBufferSize);
+    compressedBuffer = malloc(compressedBufferSize);
+    decodedBuffer = malloc(decodedBufferSize);
+    if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd)
+    {
+        DISPLAY("Not enough memory, aborting\n");
+        goto _output_error;
+    }
+    RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState);
+
+    /* Basic compression test */
+    DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZBUFF_compressInit(zc, 1);
+    readSize = CNBufferSize;
+    genSize = compressedBufferSize;
+    result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
+    if (ZBUFF_isError(result)) goto _output_error;
+    if (readSize != CNBufferSize) goto _output_error;   /* entire input should be consumed */
+    cSize = genSize;
+    genSize = compressedBufferSize - cSize;
+    result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
+    if (result != 0) goto _output_error;   /* error, or some data not flushed */
+    cSize += genSize;
+    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+
+    /* Basic decompression test */
+    DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZBUFF_decompressInit(zd);
+    readSize = cSize;
+    genSize = CNBufferSize;
+    result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
+    if (result != 0) goto _output_error;  /* should reach end of frame == 0; otherwise, some data left, or an error */
+    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
+    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* check regenerated data is byte exact */
+    {
+        size_t i;
+        DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+        for (i=0; i<CNBufferSize; i++)
+        {
+            if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
+        }
+        DISPLAYLEVEL(4, "OK \n");
+    }
+
+_end:
+    ZBUFF_freeCCtx(zc);
+    ZBUFF_freeDCtx(zd);
+    free(CNBuffer);
+    free(compressedBuffer);
+    free(decodedBuffer);
+    return testResult;
+
+_output_error:
+    testResult = 1;
+    DISPLAY("Error detected in Unit tests ! \n");
+    goto _end;
+}
+
+
+static size_t findDiff(const void* buf1, const void* buf2, size_t max)
+{
+    const BYTE* b1 = (const BYTE*)buf1;
+    const BYTE* b2 = (const BYTE*)buf2;
+    size_t i;
+    for (i=0; i<max; i++)
+    {
+        if (b1[i] != b2[i]) break;
+    }
+    return i;
+}
+
+#   define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
+                            DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
+
+static const U32 maxSrcLog = 24;
+static const U32 maxSampleLog = 19;
+
+int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
+{
+    BYTE* cNoiseBuffer[5];
+    BYTE* srcBuffer;
+    size_t srcBufferSize = (size_t)1<<maxSrcLog;
+    BYTE* copyBuffer;
+    size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
+    BYTE* cBuffer;
+    size_t cBufferSize   = ZSTD_compressBound(srcBufferSize);
+    BYTE* dstBuffer;
+    size_t dstBufferSize = srcBufferSize;
+    U32 result = 0;
+    U32 testNb = 0;
+    U32 coreSeed = seed, lseed = 0;
+    ZBUFF_CCtx* zc;
+    ZBUFF_DCtx* zd;
+    XXH64_state_t crc64;
+    U32 startTime = FUZ_GetMilliStart();
+
+    /* allocation */
+    zc = ZBUFF_createCCtx();
+    zd = ZBUFF_createDCtx();
+    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
+    cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
+    copyBuffer= (BYTE*)malloc (copyBufferSize);
+    dstBuffer = (BYTE*)malloc (dstBufferSize);
+    cBuffer   = (BYTE*)malloc (cBufferSize);
+    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
+           !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
+           "Not enough memory, fuzzer tests cancelled");
+
+    /* Create initial samples */
+    RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
+    RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
+    RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
+    RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
+    RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
+    srcBuffer = cNoiseBuffer[2];
+
+    /* catch up testNb */
+    for (testNb=1; testNb < startTest; testNb++)
+        FUZ_rand(&coreSeed);
+
+    /* test loop */
+    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
+    {
+        size_t sampleSize, sampleStart;
+        size_t cSize;
+        size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
+        size_t errorCode;
+        U32 sampleSizeLog, buffNb, n, nbChunks;
+        U64 crcOrig, crcDest;
+
+        /* init */
+        DISPLAYUPDATE(2, "\r%6u", testNb);
+        if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u   ", nbTests);
+        FUZ_rand(&coreSeed);
+        lseed = coreSeed ^ prime1;
+        buffNb = FUZ_rand(&lseed) & 127;
+        if (buffNb & 7) buffNb=2;   /* select buffer */
+        else
+        {
+            buffNb >>= 3;
+            if (buffNb & 7)
+            {
+                const U32 tnb[2] = { 1, 3 };
+                buffNb = tnb[buffNb >> 3];
+            }
+            else
+            {
+                const U32 tnb[2] = { 0, 4 };
+                buffNb = tnb[buffNb >> 3];
+            }
+        }
+        srcBuffer = cNoiseBuffer[buffNb];
+
+        /* Multi - segments compression test */
+        XXH64_reset(&crc64, 0);
+        nbChunks = (FUZ_rand(&lseed) & 127) + 2;
+        sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
+        maxTestSize = (size_t)1 << sampleSizeLog;
+        maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
+        ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
+        totalTestSize = 0;
+        cSize = 0;
+        for (n=0; n<nbChunks; n++)
+        {
+            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+            sampleSize = (size_t)1 << sampleSizeLog;
+            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+            sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+
+            XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
+            memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, sampleSize);
+
+            readSize = sampleSize;
+            genSize = cBufferSize - cSize;
+            errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
+            CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
+            CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
+            cSize += genSize;
+            totalTestSize += sampleSize;
+
+            if ((FUZ_rand(&lseed) & 15) == 0)
+            {
+                /* add a few random flushes operations, to mess around */
+                genSize = cBufferSize - cSize;
+                errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
+                CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
+                cSize += genSize;
+            }
+
+            if (totalTestSize > maxTestSize) break;
+        }
+        genSize = cBufferSize - cSize;
+        errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
+        CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
+        CHECK (errorCode != 0, "frame epilogue not fully consumed");
+        cSize += genSize;
+        crcOrig = XXH64_digest(&crc64);
+
+        /* multi - fragments decompression test */
+        ZBUFF_decompressInit(zd);
+        totalCSize = 0;
+        totalGenSize = 0;
+        while (totalCSize < cSize)
+        {
+            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
+            sampleSize  = (size_t)1 << sampleSizeLog;
+            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+            readSize = sampleSize;
+            genSize = dstBufferSize - totalGenSize;
+            errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
+            CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
+            totalGenSize += genSize;
+            totalCSize += readSize;
+        }
+        CHECK (errorCode != 0, "frame not fully decoded");
+        CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
+        CHECK (totalCSize != cSize, "compressed data should be fully read")
+        crcDest = XXH64(dstBuffer, totalTestSize, 0);
+        if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
+        CHECK (crcDest!=crcOrig, "decompressed data corrupted");
+
+        /* noisy/erroneous src decompression test */
+        /* add some noise */
+        nbChunks = (FUZ_rand(&lseed) & 7) + 2;
+        for (n=0; n<nbChunks; n++)
+        {
+            size_t cStart;
+
+            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+            sampleSize = (size_t)1 << sampleSizeLog;
+            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+            if (sampleSize > cSize/3) sampleSize = cSize/3;
+            sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+            cStart = FUZ_rand(&lseed) % (cSize - sampleSize);
+
+            memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
+        }
+
+        /* try decompression on noisy data */
+        ZBUFF_decompressInit(zd);
+        totalCSize = 0;
+        totalGenSize = 0;
+        while (totalCSize < cSize)
+        {
+            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
+            sampleSize  = (size_t)1 << sampleSizeLog;
+            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+            readSize = sampleSize;
+            genSize = dstBufferSize - totalGenSize;
+            errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
+            if (ZBUFF_isError(errorCode)) break;   /* error correctly detected */
+        }
+    }
+    DISPLAY("\r%u fuzzer tests completed   \n", testNb);
+
+_cleanup:
+    ZBUFF_freeCCtx(zc);
+    ZBUFF_freeDCtx(zd);
+    free(cNoiseBuffer[0]);
+    free(cNoiseBuffer[1]);
+    free(cNoiseBuffer[2]);
+    free(cNoiseBuffer[3]);
+    free(cNoiseBuffer[4]);
+    free(copyBuffer);
+    free(cBuffer);
+    free(dstBuffer);
+    return result;
+
+_output_error:
+    result = 1;
+    goto _cleanup;
+}
+
+
+/*********************************************************
+*  Command line
+*********************************************************/
+int FUZ_usage(char* programName)
+{
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -i#    : Nb of tests (default:%u) \n", nbTestsDefault);
+    DISPLAY( " -s#    : Select seed (default:prompt user)\n");
+    DISPLAY( " -t#    : Select starting test number (default:0)\n");
+    DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -v     : verbose\n");
+    DISPLAY( " -p     : pause at the end\n");
+    DISPLAY( " -h     : display help and exit\n");
+    return 0;
+}
+
+
+int main(int argc, char** argv)
+{
+    U32 seed=0;
+    int seedset=0;
+    int argNb;
+    int nbTests = nbTestsDefault;
+    int testNb = 0;
+    int proba = FUZ_COMPRESSIBILITY_DEFAULT;
+    int result=0;
+    U32 mainPause = 0;
+    char* programName;
+
+    /* Check command line */
+    programName = argv[0];
+    for(argNb=1; argNb<argc; argNb++)
+    {
+        char* argument = argv[argNb];
+
+        if(!argument) continue;   /* Protection if argument empty */
+
+        /* Handle commands. Aggregated commands are allowed */
+        if (argument[0]=='-')
+        {
+            argument++;
+
+            while (*argument!=0)
+            {
+                switch(*argument)
+                {
+                case 'h':
+                    return FUZ_usage(programName);
+                case 'v':
+                    argument++;
+                    g_displayLevel=4;
+                    break;
+                case 'q':
+                    argument++;
+                    g_displayLevel--;
+                    break;
+                case 'p': /* pause at the end */
+                    argument++;
+                    mainPause = 1;
+                    break;
+
+                case 'i':
+                    argument++;
+                    nbTests=0; g_testTime=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        nbTests *= 10;
+                        nbTests += *argument - '0';
+                        argument++;
+                    }
+                    break;
+
+                case 'T':
+                    argument++;
+                    nbTests=0; g_testTime=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        g_testTime *= 10;
+                        g_testTime += *argument - '0';
+                        argument++;
+                    }
+                    if (*argument=='m') g_testTime *=60, argument++;
+                    if (*argument=='n') argument++;
+                    g_testTime *= 1000;
+                    break;
+
+                case 's':
+                    argument++;
+                    seed=0;
+                    seedset=1;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        seed *= 10;
+                        seed += *argument - '0';
+                        argument++;
+                    }
+                    break;
+
+                case 't':
+                    argument++;
+                    testNb=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        testNb *= 10;
+                        testNb += *argument - '0';
+                        argument++;
+                    }
+                    break;
+
+                case 'P':   /* compressibility % */
+                    argument++;
+                    proba=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        proba *= 10;
+                        proba += *argument - '0';
+                        argument++;
+                    }
+                    if (proba<0) proba=0;
+                    if (proba>100) proba=100;
+                    break;
+
+                default:
+                    return FUZ_usage(programName);
+                }
+            }
+        }
+    }
+
+    /* Get Seed */
+    DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
+
+    if (!seedset) seed = FUZ_GetMilliStart() % 10000;
+    DISPLAY("Seed = %u\n", seed);
+    if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
+
+    if (nbTests<=0) nbTests=1;
+
+    if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100);  /* constant seed for predictability */
+    if (!result)
+        result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
+    if (mainPause)
+    {
+        int unused;
+        DISPLAY("Press Enter \n");
+        unused = getchar();
+        (void)unused;
+    }
+    return result;
+}

programs/zstdcli.c

@@ -70,7 +70,7 @@
 **************************************/
 #define COMPRESSOR_NAME "zstd command line interface"
 #ifndef ZSTD_VERSION
-#  define ZSTD_VERSION "v0.3.6"
+#  define ZSTD_VERSION "v0.4.0"
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__
@@ -163,7 +163,7 @@ static void waitEnter(void)
 }
 
 
-int main(int argc, char** argv)
+int main(int argCount, const char** argv)
 {
     int i,
         bench=0,
@@ -196,9 +196,9 @@ int main(int argc, char** argv)
         decode=1;
 
     /* command switches */
-    for(i=1; i<argc; i++)
+    for(i=1; i<argCount; i++)
     {
-        char* argument = argv[i];
+        const char* argument = argv[i];
 
         if(!argument) continue;   /* Protection if argument empty */
 
@@ -311,7 +311,7 @@ int main(int argc, char** argv)
         }
 
         /* first provided filename is input */
-        if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argc-i; continue; }
+        if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argCount-i; continue; }
 
         /* second provided filename is output */
         if (!outFileName)

visual/2012/fullbench/fullbench.vcxproj

@@ -73,7 +73,7 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <IncludePath>$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
     <RunCodeAnalysis>true</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -95,7 +95,7 @@
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>true</EnablePREfast>
-      <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -163,7 +163,8 @@
     <ClCompile Include="..\..\..\lib\huff0.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
-    <ClCompile Include="..\..\..\lib\zstd.c" />
+    <ClCompile Include="..\..\..\lib\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\zstd_decompress.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\programs\fullbench.c" />
   </ItemGroup>

visual/2012/fullbench/fullbench.vcxproj.filters

@@ -18,9 +18,6 @@
     <ClCompile Include="..\..\..\lib\fse.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\lib\zstd.c">
-      <Filter>Fichiers sources</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\programs\fullbench.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
@@ -36,6 +33,12 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\lib\zstd_compress.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\zstd_decompress.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\fse.h">

visual/2012/fuzzer/fuzzer.vcxproj

@@ -73,7 +73,7 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <IncludePath>$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
     <RunCodeAnalysis>true</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -95,7 +95,7 @@
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>true</EnablePREfast>
-      <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -163,8 +163,8 @@
     <ClCompile Include="..\..\..\lib\huff0.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
-    <ClCompile Include="..\..\..\lib\zstd.c" />
+    <ClCompile Include="..\..\..\lib\zstd_compress.c" />
-    <ClCompile Include="..\..\..\lib\zstdhc.c" />
+    <ClCompile Include="..\..\..\lib\zstd_decompress.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\programs\fuzzer.c" />
     <ClCompile Include="..\..\..\programs\xxhash.c" />
@@ -178,8 +178,6 @@
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
-    <ClInclude Include="..\..\..\lib\zstdhc.h" />
-    <ClInclude Include="..\..\..\lib\zstdhc_static.h" />
     <ClInclude Include="..\..\..\lib\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\zstd_static.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />

visual/2012/fuzzer/fuzzer.vcxproj.filters

@@ -18,9 +18,6 @@
     <ClCompile Include="..\..\..\lib\fse.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\lib\zstd.c">
-      <Filter>Fichiers sources</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\programs\fuzzer.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
@@ -36,10 +33,13 @@
     <ClCompile Include="..\..\..\lib\huff0.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\lib\zstdhc.c">
+    <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
+    <ClCompile Include="..\..\..\lib\zstd_compress.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\zstd_decompress.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
   </ItemGroup>
@@ -74,12 +74,6 @@
     <ClInclude Include="..\..\..\lib\zstd_internal.h">
       <Filter>Fichiers d%27en-tête</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\..\lib\zstdhc.h">
-      <Filter>Fichiers d%27en-tête</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\..\lib\zstdhc_static.h">
-      <Filter>Fichiers d%27en-tête</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
       <Filter>Fichiers d%27en-tête</Filter>
     </ClInclude>

visual/2012/zstd/zstd.vcxproj

@@ -23,8 +23,9 @@
     <ClCompile Include="..\..\..\lib\huff0.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
-    <ClCompile Include="..\..\..\lib\zstd.c" />
+    <ClCompile Include="..\..\..\lib\zstd_buffered.c" />
-    <ClCompile Include="..\..\..\lib\zstdhc.c" />
+    <ClCompile Include="..\..\..\lib\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\zstd_decompress.c" />
     <ClCompile Include="..\..\..\programs\bench.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
     <ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
@@ -40,8 +41,8 @@
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
-    <ClInclude Include="..\..\..\lib\zstdhc.h" />
+    <ClInclude Include="..\..\..\lib\zstd_buffered.h" />
-    <ClInclude Include="..\..\..\lib\zstdhc_static.h" />
+    <ClInclude Include="..\..\..\lib\zstd_buffered_static.h" />
     <ClInclude Include="..\..\..\lib\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\zstd_static.h" />
     <ClInclude Include="..\..\..\programs\bench.h" />
@@ -104,7 +105,7 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
     <RunCodeAnalysis>true</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -115,7 +116,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
     <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
-    <RunCodeAnalysis>true</RunCodeAnalysis>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -126,7 +127,7 @@
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>true</EnablePREfast>
-      <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -140,7 +141,7 @@
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <TreatWarningAsError>true</TreatWarningAsError>
+      <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>true</EnablePREfast>
       <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
@@ -178,9 +179,8 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <TreatWarningAsError>true</TreatWarningAsError>
+      <TreatWarningAsError>false</TreatWarningAsError>
-      <EnablePREfast>true</EnablePREfast>
+      <EnablePREfast>false</EnablePREfast>
-      <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
     </ClCompile>
     <Link>

visual/2012/zstd/zstd.vcxproj.filters

@@ -18,9 +18,6 @@
     <ClCompile Include="..\..\..\lib\fse.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\lib\zstd.c">
-      <Filter>Fichiers sources</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\programs\bench.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
@@ -39,15 +36,21 @@
     <ClCompile Include="..\..\..\lib\huff0.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\lib\zstdhc.c">
-      <Filter>Fichiers sources</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
     <ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c">
       <Filter>Fichiers sources</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\lib\zstd_compress.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\zstd_decompress.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\zstd_buffered.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\fse.h">
@@ -83,12 +86,6 @@
     <ClInclude Include="..\..\..\lib\zstd_internal.h">
       <Filter>Fichiers d%27en-tête</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\..\lib\zstdhc.h">
-      <Filter>Fichiers d%27en-tête</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\..\lib\zstdhc_static.h">
-      <Filter>Fichiers d%27en-tête</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
       <Filter>Fichiers d%27en-tête</Filter>
     </ClInclude>
@@ -98,5 +95,11 @@
     <ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h">
       <Filter>Fichiers d%27en-tête</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\lib\zstd_buffered.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\zstd_buffered_static.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>