Merge remote-tracking branch 'refs/remotes/origin/playTests' into dev

2016-05-24 15:38:52 +02:00 · 2016-05-24 15:38:52 +02:00 · 4a9ec65c9d
commit 4a9ec65c9d
parent 5e334be457 71dd67cf57
23 changed files with 914 additions and 184 deletions
--- a/.gitattributes
+++ b/.gitattributes
@ -14,3 +14,6 @@
 *.vcxproj* text eol=crlf
 *.suo binary
 *.rc binary
+
+# Windows
+*.bat text eol=crlf
--- a/appveyor.yml
+++ b/appveyor.yml
@ -1,52 +1,42 @@
 version: 1.0.{build}
 environment:
  matrix:
-  - ZSTD_APPVEYOR: "gcc"
-  - ZSTD_APPVEYOR: "visual"
-    CONFIGURATION: "Debug"
-    PLATFORM: "Win32"
-  - ZSTD_APPVEYOR: "visual"
+  - COMPILER: "visual"
    CONFIGURATION: "Release"
    PLATFORM: "Win32"
-  - ZSTD_APPVEYOR: "visual"
-    CONFIGURATION: "Debug"
-    PLATFORM: "x64"
-  - ZSTD_APPVEYOR: "visual"
+  - COMPILER: "visual"
    CONFIGURATION: "Release"
    PLATFORM: "x64"
+  - COMPILER: "visual"
+    CONFIGURATION: "Debug"
+    PLATFORM: "Win32"
+  - COMPILER: "visual"
+    CONFIGURATION: "Debug"
+    PLATFORM: "x64"
+  - COMPILER: "gcc"
+    MAKE_PARAMS: "test"
+    PLATFORM: "mingw32"
+  - COMPILER: "gcc"
+    MAKE_PARAMS: "test"
+    PLATFORM: "mingw64"

 install:
-  - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH_ORIGINAL=%PATH%
-  - if [%ZSTD_APPVEYOR%]==[gcc] (
-      SET "MINGW_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen" &&
+  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
+  - if [%COMPILER%]==[gcc] (
      SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
      SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
      SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
      COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe
    ) else (
-      IF [%PLATFORM%]==[x64] SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+      IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
    )

 build_script:
-  - ECHO Building %ZSTD_APPVEYOR%
-  - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% 
-  - if [%ZSTD_APPVEYOR%]==[gcc] (
-      ECHO *** &&
-      ECHO *** Building mingw32 &&
-      ECHO *** &&
-      ECHO make %MINGW_PARAMS% &&
-      make %MINGW_PARAMS% &&
-      COPY programs\fuzzer.exe projects\fuzzer_mingw32.exe &&
-      make clean
-    )
-  - if [%ZSTD_APPVEYOR%]==[gcc] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% 
-  - if [%ZSTD_APPVEYOR%]==[gcc] (
-      ECHO *** &&
-      ECHO *** Building mingw64 &&
-      ECHO *** &&
-      ECHO make %MINGW_PARAMS% &&
-      make %MINGW_PARAMS% &&
-      COPY programs\fuzzer.exe projects\fuzzer_mingw64.exe &&
+  - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% 
+  - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% 
+  - if [%PLATFORM%]==[mingw64] (
      make clean &&
      ECHO *** &&
      ECHO *** Building clang &&
@ -56,7 +46,15 @@ build_script:
      COPY programs\fuzzer.exe projects\fuzzer_clang.exe &&
      make clean
    )
-  - if [%ZSTD_APPVEYOR%]==[visual] (
+  - if [%COMPILER%]==[gcc] (
+      ECHO *** &&
+      ECHO *** Building %PLATFORM% &&
+      ECHO *** &&
+      ECHO make %MAKE_PARAMS% &&
+      make %MAKE_PARAMS% &&
+      make clean
+    )
+  - if [%COMPILER%]==[visual] (
      ECHO *** &&
      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% &&
      ECHO *** &&
@ -65,7 +63,7 @@ build_script:
      ECHO *** &&
      ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
      ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:normal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
      ECHO *** &&
      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
@ -80,18 +78,25 @@ build_script:
      ECHO *** &&
      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
      ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:normal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe
+      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\
    )

 test_script:
+  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
  - SET FUZZERTEST=-T1mn
-  - if [%ZSTD_APPVEYOR%]==[gcc] (
-      projects\fuzzer_mingw32.exe %FUZZERTEST% &&
-      projects\fuzzer_mingw64.exe %FUZZERTEST% &&
-      projects\fuzzer_clang.exe %FUZZERTEST%
+  - if [%COMPILER%]==[gcc] (
+      if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST%
    )
-  - if [%ZSTD_APPVEYOR%]==[visual] if [%CONFIGURATION%]==[Release] (
+  - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
+      CD programs &&
+      SET ZSTD=..\projects\zstd.exe &&
+      SET DATAGEN=..\projects\datagen.exe &&
+      CALL tests\playTests.bat --test-large-data &&
+      CD .. &&
+      projects\fullbench.exe -i1 &&
+      projects\fullbench.exe -i1 -P0 &&
      projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
      projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
      projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@ -42,8 +42,11 @@ extern "C" {
 /*-****************************************
 *  Dependencies
 ******************************************/
-#include <stddef.h>    /* size_t, ptrdiff_t */
-#include <string.h>    /* memcpy */
+#include <stddef.h>     /* size_t, ptrdiff_t */
+#include <string.h>     /* memcpy */
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#endif


 /*-****************************************
--- a/lib/common/zbuff.h
+++ b/lib/common/zbuff.h
@ -38,7 +38,8 @@ extern "C" {
 /* *************************************
 *  Dependencies
 ***************************************/
-#include <stddef.h>   /* size_t */
+#include <stddef.h>      /* size_t */
+#include "zstd_static.h" /* ZSTD_customMem */


 /* ***************************************************************
@ -60,6 +61,7 @@ extern "C" {
 ***************************************/
 typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
 ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
+ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t      ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);

 ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
@ -112,6 +114,7 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap

 typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
 ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
+ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t      ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);

 ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@ -186,6 +186,8 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val)
 /*-*******************************************
 *  Private interfaces
 *********************************************/
+typedef struct ZSTD_stats_s ZSTD_stats_t;
+
 typedef struct {
    U32 off;
    U32 len;
@ -199,16 +201,6 @@ typedef struct {
    U32 rep[ZSTD_REP_INIT];
 } ZSTD_optimal_t;

-#if ZSTD_OPT_DEBUG == 3
-    #include ".debug/zstd_stats.h"
-#else
-    typedef struct { U32  unused; } ZSTD_stats_t;
-    MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
-    MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
-    MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
-    MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
-#endif
-
 typedef struct {
    void* buffer;
    U32*  offsetStart;
@ -245,7 +237,7 @@ typedef struct {
    U32  cachedPrice;
    U32  cachedLitLength;
    const BYTE* cachedLiterals;
-    ZSTD_stats_t stats;
+    ZSTD_stats_t* stats;
 } seqStore_t;

 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
--- a/lib/common/zstd_static.h
+++ b/lib/common/zstd_static.h
@ -94,10 +94,22 @@ typedef struct {
    ZSTD_frameParameters fParams;
 } ZSTD_parameters;

+typedef void* (*ZSTD_allocFunction) (size_t size);
+typedef void  (*ZSTD_freeFunction) (void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem;
+

 /*-*************************************
 *  Advanced functions
 ***************************************/
+/*! ZSTD_createCCtx_advanced() :
+ *  Create a ZSTD compression context using external alloc and free functions */ 
+ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+
+/*! ZSTD_createDCtx_advanced() :
+ *  Create a ZSTD decompression context using external alloc and free functions */ 
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+
 ZSTDLIB_API unsigned ZSTD_maxCLevel (void);

 /*! ZSTD_getCParams() :
--- a/lib/compress/.debug/zstd_stats.h
+++ b/lib/compress/.debug/zstd_stats.h
@ -37,57 +37,52 @@
 extern "C" {
 #endif

-/*-*************************************
-*  Dependencies
-***************************************/
-//#include "zstd.h"
-//#include "mem.h"
-
-
-/*-*************************************
-*  Constants
-***************************************/
-//#define ZSTD_MAGICNUMBER 0xFD2FB526   /* v0.6 */

+#if ZSTD_OPT_DEBUG == 3

 /*-*************************************
 *  Types
 ***************************************/
-typedef struct {
-    U32  priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength;
+struct ZSTD_stats_s {
+    U32  priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength;
    U32  totalMatchSum, totalLitSum, totalSeqSum, totalRepSum;
    U32  litSum, matchLengthSum, litLengthSum, offCodeSum;
-    U32  matchLengthFreq[1<<MLbits];
-    U32  litLengthFreq[1<<LLbits];
+    U32  matchLengthFreq[MaxML+1];
+    U32  litLengthFreq[MaxLL+1];
    U32  litFreq[1<<Litbits];
-    U32  offCodeFreq[1<<Offbits];
-} ZSTD_stats_t;
+    U32  offCodeFreq[MaxOff+1];
+};


 /*-*************************************
-*  Advanced functions
+*  Stats functions
 ***************************************/
+MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); }
+MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); }
+
 MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength)
 {
    stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4);
-    printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
-    printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8);
+    printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
+    printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8);
 }

+
 MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats)
 {
    stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1;
-    stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0;
+    stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0;
 }

+
 MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
 {
    unsigned u;

-    stats->litSum = (1<<Litbits);
-    stats->litLengthSum = (1<<LLbits);
-    stats->matchLengthSum = (1<<MLbits);
-    stats->offCodeSum = (1<<Offbits);
+    stats->litSum = (2<<Litbits);
+    stats->litLengthSum = MaxLL+1;
+    stats->matchLengthSum = MaxML+1;
+    stats->offCodeSum = (MaxOff+1);

    for (u=0; u<=MaxLit; u++)
        stats->litFreq[u] = 1;
@ -99,62 +94,77 @@ MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
        stats->offCodeFreq[u] = 1;
 }

+
 MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
 {
-    /* offset */
-    BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
-    stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
-    stats->priceOffset += (offCode-1) + (!offCode);
-
-    /* match Length */
-    stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3);
-    stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1);
-
-    if (litLength) {
-        /* literals */
-        U32 u;
-        stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
-        for (u=0; u < litLength; u++)
-            stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
-
-        /* literal Length */
-        stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3);
-        stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1);
-    } else {
-        stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
-    }
-
-
-    if (offset == 0) stats->totalRepSum++;
-    stats->totalSeqSum++;
-    stats->totalMatchSum += matchLength;
-    stats->totalLitSum += litLength;
-        
    U32 u;
    /* literals */
+    stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
+    for (u=0; u < litLength; u++)
+        stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
    stats->litSum += litLength;
    for (u=0; u < litLength; u++)
        stats->litFreq[literals[u]]++;

    /* literal Length */
-    stats->litLengthSum++;
-    if (litLength >= MaxLL)
-        stats->litLengthFreq[MaxLL]++;
-    else
-        stats->litLengthFreq[litLength]++;
+    {   static const BYTE LL_Code[64] = {  0,  1,  2,  3,  4,  5,  6,  7,
+                                           8,  9, 10, 11, 12, 13, 14, 15,
+                                          16, 16, 17, 17, 18, 18, 19, 19,
+                                          20, 20, 20, 20, 21, 21, 21, 21,
+                                          22, 22, 22, 22, 22, 22, 22, 22,
+                                          23, 23, 23, 23, 23, 23, 23, 23,
+                                          24, 24, 24, 24, 24, 24, 24, 24,
+                                          24, 24, 24, 24, 24, 24, 24, 24 };
+        const BYTE LL_deltaCode = 19;
+        const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
+        if (litLength) {
+            stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1);
+        } else {
+            stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
+        }
+        stats->litLengthFreq[llCode]++;
+        stats->litLengthSum++;
+    }

    /* match offset */
-    stats->offCodeSum++;
-    stats->offCodeFreq[offCode]++;
+    {   BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
+        stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
+        stats->priceOffset += offCode;
+        stats->offCodeSum++;
+        stats->offCodeFreq[offCode]++;
+    }

    /* match Length */
-    stats->matchLengthSum++;
-    if (matchLength >= MaxML)
-        stats->matchLengthFreq[MaxML]++;
-    else
-        stats->matchLengthFreq[matchLength]++;
+    {   static const BYTE ML_Code[128] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
+                                          16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+                                          32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
+                                          38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
+                                          40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+                                          41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+                                          42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+                                          42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
+        const BYTE ML_deltaCode = 36;
+        const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
+        stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1);
+        stats->matchLengthFreq[mlCode]++;
+        stats->matchLengthSum++;
+    }
+
+    if (offset == 0) stats->totalRepSum++;
+    stats->totalSeqSum++;
+    stats->totalMatchSum += matchLength;
+    stats->totalLitSum += litLength;
 }

+#else
+    struct ZSTD_stats_s { U32 unused; };
+    MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; }
+    MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; }
+    MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
+    MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
+    MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
+    MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
+#endif // #if ZSTD_OPT_DEBUG == 3


 #if defined (__cplusplus)
--- a/lib/compress/zbuff_compress.c
+++ b/lib/compress/zbuff_compress.c
@ -95,14 +95,39 @@ struct ZBUFF_CCtx_s {
    size_t outBuffContentSize;
    size_t outBuffFlushedSize;
    ZBUFF_cStage stage;
+    ZSTD_allocFunction customAlloc;
+    ZSTD_freeFunction customFree;
 };   /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */

 ZBUFF_CCtx* ZBUFF_createCCtx(void)
 {
-    ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
+    ZSTD_customMem customMem = { NULL, NULL };
+    return ZBUFF_createCCtx_advanced(customMem);
+}
+
+ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
+{
+    ZBUFF_CCtx* zbc;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+    {
+        zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx));
+        if (zbc==NULL) return NULL;
+        zbc->customAlloc = malloc;
+        zbc->customFree = free;
+        zbc->zc = ZSTD_createCCtx();
+        return zbc;
+    }
+
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx));
    if (zbc==NULL) return NULL;
-    memset(zbc, 0, sizeof(*zbc));
-    zbc->zc = ZSTD_createCCtx();
+    memset(zbc, 0, sizeof(ZBUFF_CCtx));
+    zbc->customAlloc = customMem.customAlloc; 
+    zbc->customFree = customMem.customFree;
+    zbc->zc = ZSTD_createCCtx_advanced(customMem);
    return zbc;
 }

@ -110,9 +135,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
 {
    if (zbc==NULL) return 0;   /* support free on NULL */
    ZSTD_freeCCtx(zbc->zc);
-    free(zbc->inBuff);
-    free(zbc->outBuff);
-    free(zbc);
+    zbc->customFree(zbc->inBuff);
+    zbc->customFree(zbc->outBuff);
+    zbc->customFree(zbc);
    return 0;
 }

@ -127,16 +152,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
    {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
        if (zbc->inBuffSize < neededInBuffSize) {
            zbc->inBuffSize = neededInBuffSize;
-            free(zbc->inBuff);   /* should not be necessary */
-            zbc->inBuff = (char*)malloc(neededInBuffSize);
+            zbc->customFree(zbc->inBuff);   /* should not be necessary */
+            zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize);
            if (zbc->inBuff == NULL) return ERROR(memory_allocation);
        }
        zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
    }
    if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
        zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
-        free(zbc->outBuff);   /* should not be necessary */
-        zbc->outBuff = (char*)malloc(zbc->outBuffSize);
+        zbc->customFree(zbc->outBuff);   /* should not be necessary */
+        zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize);
        if (zbc->outBuff == NULL) return ERROR(memory_allocation);
    }

--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@ -57,6 +57,7 @@
 #include "fse_static.h"
 #include "huf_static.h"
 #include "zstd_internal.h"
+#include ".debug/zstd_stats.h"


 /*-*************************************
@ -103,6 +104,8 @@ struct ZSTD_CCtx_s
    void* workSpace;
    size_t workSpaceSize;
    size_t blockSize;
+    ZSTD_allocFunction customAlloc;
+    ZSTD_freeFunction customFree;

    seqStore_t seqStore;    /* sequences storage ptrs */
    U32* hashTable;
@ -117,13 +120,40 @@ struct ZSTD_CCtx_s

 ZSTD_CCtx* ZSTD_createCCtx(void)
 {
-    return (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
+    ZSTD_customMem customMem = { NULL, NULL };
+    return ZSTD_createCCtx_advanced(customMem);
+}
+
+ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
+{
+    ZSTD_CCtx* ctx;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+    {
+        ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
+        if (!ctx) return NULL;
+
+        ctx->customAlloc = malloc;
+        ctx->customFree = free;
+        return ctx;
+    }
+
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx));
+    if (!ctx) return NULL;
+
+    memset(ctx, 0, sizeof(ZSTD_CCtx));
+    ctx->customAlloc = customMem.customAlloc; 
+    ctx->customFree = customMem.customFree;
+    return ctx;
 }

 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 {
-    free(cctx->workSpace);
-    free(cctx);
+    cctx->customFree(cctx->workSpace);
+    cctx->customFree(cctx);
    return 0;   /* reserved as a potential error code in the future */
 }

@ -228,8 +258,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
        size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
                              + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
        if (zc->workSpaceSize < neededSpace) {
-            free(zc->workSpace);
-            zc->workSpace = malloc(neededSpace);
+            zc->customFree(zc->workSpace);
+            zc->workSpace = zc->customAlloc(neededSpace);
            if (zc->workSpace == NULL) return ERROR(memory_allocation);
            zc->workSpaceSize = neededSpace;
    }   }
@ -285,7 +315,9 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
 {
    if (srcCCtx->stage!=1) return ERROR(stage_wrong);

-    dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
+    dstCCtx->hashLog3    = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
+    dstCCtx->customAlloc = srcCCtx->customAlloc;
+    dstCCtx->customFree  = srcCCtx->customFree;
    ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0);
    dstCCtx->params.fParams.contentSizeFlag = 0;   /* content size different from the one set during srcCCtx init */

@ -856,7 +888,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
        printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
               pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
 #endif
-    ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode);
+    ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode);

    /* copy Literals */
    ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@ -1748,7 +1780,7 @@ _storeSequence:
    {   size_t const lastLLSize = iend - anchor;
        memcpy(seqStorePtr->lit, anchor, lastLLSize);
        seqStorePtr->lit += lastLLSize;
-        ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0);
+        ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0);
    }
 }

@ -2020,15 +2052,15 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
    BYTE* const ostart = (BYTE*)dst;
    BYTE* op = ostart;
    const U32 maxDist = 1 << zc->params.cParams.windowLog;
-    ZSTD_stats_t* stats = &zc->seqStore.stats;
-
+    ZSTD_stats_t* stats = ZSTD_statsAlloc();
+    zc->seqStore.stats = stats;
    ZSTD_statsInit(stats);

    while (remaining) {
        size_t cSize;
        ZSTD_statsResetFreqs(stats);

-        if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
+        if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); }   /* not enough space to store compressed block */
        if (remaining < blockSize) blockSize = remaining;

        if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
@ -2039,11 +2071,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
        }

        cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
-        if (ZSTD_isError(cSize)) return cSize;
+        if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; }

        if (cSize == 0) {  /* block is not compressible */
            cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize);
-            if (ZSTD_isError(cSize)) return cSize;
+            if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; }
        } else {
            op[0] = (BYTE)(cSize>>16);
            op[1] = (BYTE)(cSize>>8);
@ -2059,6 +2091,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
    }

    ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
+    ZSTD_statsFree(stats);
    return op-ostart;
 }

@ -2424,8 +2457,10 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
    size_t result;
    ZSTD_CCtx ctxBody;
    memset(&ctxBody, 0, sizeof(ctxBody));
+    ctxBody.customAlloc = malloc;
+    ctxBody.customFree = free;
    result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
-    free(ctxBody.workSpace);   /* can't free ctxBody, since it's on stack; just free heap content */
+    ctxBody.customFree(ctxBody.workSpace);   /* can't free ctxBody, since it's on stack; just free heap content */
    return result;
 }

--- a/lib/decompress/zbuff_decompress.c
+++ b/lib/decompress/zbuff_decompress.c
@ -82,15 +82,41 @@ struct ZBUFF_DCtx_s {
    size_t blockSize;
    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
    size_t lhSize;
+    ZSTD_allocFunction customAlloc;
+    ZSTD_freeFunction customFree;
 };   /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */


 ZBUFF_DCtx* ZBUFF_createDCtx(void)
 {
-    ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
+    ZSTD_customMem customMem = { NULL, NULL };
+    return ZBUFF_createDCtx_advanced(customMem);
+}
+
+ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
+{
+    ZBUFF_DCtx* zbd;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+    {
+        zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx));
+        if (zbd==NULL) return NULL;
+        zbd->customAlloc = malloc;
+        zbd->customFree = free;
+        zbd->zd = ZSTD_createDCtx();
+        zbd->stage = ZBUFFds_init;
+        return zbd;
+    }
+
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx));
    if (zbd==NULL) return NULL;
-    memset(zbd, 0, sizeof(*zbd));
-    zbd->zd = ZSTD_createDCtx();
+    memset(zbd, 0, sizeof(ZBUFF_DCtx));
+    zbd->customAlloc = customMem.customAlloc; 
+    zbd->customFree = customMem.customFree;
+    zbd->zd = ZSTD_createDCtx_advanced(customMem);
    zbd->stage = ZBUFFds_init;
    return zbd;
 }
@ -99,9 +125,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
 {
    if (zbd==NULL) return 0;   /* support free on null */
    ZSTD_freeDCtx(zbd->zd);
-    free(zbd->inBuff);
-    free(zbd->outBuff);
-    free(zbd);
+    zbd->customFree(zbd->inBuff);
+    zbd->customFree(zbd->outBuff);
+    zbd->customFree(zbd);
    return 0;
 }

@ -170,16 +196,16 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
            {   size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
                zbd->blockSize = blockSize;
                if (zbd->inBuffSize < blockSize) {
-                    free(zbd->inBuff);
+                    zbd->customFree(zbd->inBuff);
                    zbd->inBuffSize = blockSize;
-                    zbd->inBuff = (char*)malloc(blockSize);
+                    zbd->inBuff = (char*)zbd->customAlloc(blockSize);
                    if (zbd->inBuff == NULL) return ERROR(memory_allocation);
                }
                {   size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
                    if (zbd->outBuffSize < neededOutSize) {
-                        free(zbd->outBuff);
+                        zbd->customFree(zbd->outBuff);
                        zbd->outBuffSize = neededOutSize;
-                        zbd->outBuff = (char*)malloc(neededOutSize);
+                        zbd->outBuff = (char*)zbd->customAlloc(neededOutSize);
                        if (zbd->outBuff == NULL) return ERROR(memory_allocation);
            }   }   }
            zbd->stage = ZBUFFds_read;
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@ -116,6 +116,8 @@ struct ZSTD_DCtx_s
    size_t expected;
    size_t headerSize;
    ZSTD_frameParams fParams;
+    ZSTD_allocFunction customAlloc;
+    ZSTD_freeFunction customFree;
    blockType_t bType;   /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
    ZSTD_dStage stage;
    U32 flagRepeatTable;
@ -143,15 +145,41 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)

 ZSTD_DCtx* ZSTD_createDCtx(void)
 {
-    ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx));
-    if (dctx==NULL) return NULL;
+    ZSTD_customMem customMem = { NULL, NULL };
+    return ZSTD_createDCtx_advanced(customMem);
+}
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+{
+    ZSTD_DCtx* dctx;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+    {
+        dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx));
+        if (!dctx) return NULL;
+        dctx->customAlloc = malloc;
+        dctx->customFree = free;
+
+        ZSTD_decompressBegin(dctx);
+        return dctx;
+    }
+
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx));
+    if (!dctx) return NULL;
+    dctx->customAlloc = customMem.customAlloc; 
+    dctx->customFree = customMem.customFree;
+
    ZSTD_decompressBegin(dctx);
    return dctx;
 }

+
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
-    free(dctx);
+    dctx->customFree(dctx);
    return 0;   /* reserved as a potential error code in the future */
 }

--- a/programs/Makefile
+++ b/programs/Makefile
@ -163,7 +163,11 @@ clean:

 #------------------------------------------------------------------------
 #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
+#------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
+HOST_OS = POSIX
+zstd-playTests: datagen
+	ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST)

 install: zstd
 	@echo Installing binaries
@ -186,15 +190,41 @@ uninstall:
 	[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
 	@echo zstd programs successfully uninstalled

+valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
+valgrindTest: zstd datagen fuzzer fullbench zbufftest
+	@echo "\n ---- valgrind tests : memory analyzer ----"
+	$(VALGRIND) ./datagen -g50M > $(VOID)
+	$(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
+	./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID)
+	./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID)
+	./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp
+	$(VALGRIND) ./zstd -vdf tmp -o $(VOID)
+	./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID)
+	@rm tmp
+	$(VALGRIND) ./fuzzer -T1mn -t1
+	$(VALGRIND) ./fullbench -i1
+	$(VALGRIND) ./zbufftest -T1mn
+
+endif
+
+
+ifneq (,$(filter MSYS%,$(shell uname)))
+HOST_OS = MSYS
+zstd-playTests: datagen
+	ZSTD=$(ZSTD) ./tests/playTestsMSYS.sh $(ZSTDRTTEST)
+endif
+
+
+#------------------------------------------------------------------------
+#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+#------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 test: test-zstd test-fullbench test-fuzzer test-zbuff

 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32

 test-all: test test32 valgrindTest

-zstd-playTests: datagen
-	ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
-
 test-zstd: ZSTD = ./zstd
 test-zstd: zstd zstd-playTests

@ -223,20 +253,4 @@ test-zbuff: zbufftest

 test-zbuff32: zbufftest32
 	./zbufftest32 $(ZBUFFTEST)
-
-valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
-valgrindTest: zstd datagen fuzzer fullbench zbufftest
-	@echo "\n ---- valgrind tests : memory analyzer ----"
-	$(VALGRIND) ./datagen -g50M > $(VOID)
-	$(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
-	./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID)
-	./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID)
-	./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp
-	$(VALGRIND) ./zstd -vdf tmp -o $(VOID)
-	./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID)
-	@rm tmp
-	$(VALGRIND) ./fuzzer -T1mn -t1
-	$(VALGRIND) ./fullbench -i1
-	$(VALGRIND) ./zbufftest -T1mn
-
 endif
--- a/programs/datagencli.c
+++ b/programs/datagencli.c
@ -27,8 +27,8 @@
 /*-************************************
 *  Includes
 **************************************/
+#include "util.h"      /* Compiler options */
 #include <stdio.h>     /* fprintf, stderr */
-#include "mem.h"
 #include "datagen.h"   /* RDG_generate */


--- a/programs/fileio.c
+++ b/programs/fileio.c
@ -71,7 +71,7 @@
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
 #  include <fcntl.h>    /* _O_BINARY */
 #  include <io.h>       /* _setmode, _isatty */
-#  define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
+#  define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
 #else
 #  include <unistd.h>   /* isatty */
 #  define SET_BINARY_MODE(file)
@ -439,6 +439,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
    if (!strcmp(suffix, stdoutmark)) {
        unsigned u;
        ress.dstFile = stdout;
+        SET_BINARY_MODE(stdout);
        for (u=0; u<nbFiles; u++)
            missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
                                                         inFileNamesTable[u], compressionLevel);
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@ -213,6 +213,19 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2,
 /*_*******************************************************
 *  Bench functions
 *********************************************************/
+void* BMK_allocFunction(size_t size)
+{
+    void* address = malloc(size);
+   /* printf("alloc %p, %d \n", address, (int)size); */
+    return address;
+}
+
+void BMK_freeFunction(void* address)
+{
+/*   printf("free %p \n", address); */
+    free(address);
+}
+
 static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
 {
    BYTE*  dstBuff;
@ -220,6 +233,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
    BYTE*  buff2;
    const char* benchName;
    size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
+    ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction };
    double bestTime = 100000000.;

    /* Selection */
@ -247,8 +261,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
        benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
        break;
    case 42:
+        benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue";
+        break;
+    case 43:
        benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
        break;
+    case 44:
+        benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue";
+        break;
    default :
        return 0;
    }
@ -323,9 +343,16 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
        if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx();
        break;
    case 42 :
+        if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem);
+        break;
+    case 43 :
        if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx();
        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
        break;
+    case 44 :
+        if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem);
+        g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
+        break;

    /* test functions */
    /* by convention, test functions can be added > 100 */
@ -355,7 +382,29 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
        averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
        if (averageTime < bestTime) bestTime = averageTime;
        DISPLAY("%2i- %-30.30s : %7.1f MB/s  (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
-    }}
+    }
+
+    /* free allocated structures */
+    switch(benchNb)
+    {
+    case 11 :
+        if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; }
+        break;
+    case 12 :
+    case 31:
+    case 32:
+        if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; }
+        break;
+    case 41 :
+    case 42 :
+        if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; }
+        break;
+    case 43 :
+    case 44 :
+        if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; }
+        break;
+    default : ;
+    } }
    DISPLAY("%2u\n", benchNb);

 _cleanOut:
--- a/programs/tests/playTests.bat
+++ b/programs/tests/playTests.bat
@ -0,0 +1,155 @@
+@echo off
+ECHO Start playTests.bat
+ECHO ZSTD^: %ZSTD%
+ECHO DATAGEN^: %DATAGEN%
+if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1
+if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1
+
+SET ROUNDTRIPTEST=tests\roundTripTest.bat
+
+echo. && echo **** simple tests ****
+%DATAGEN% > tmp
+%ZSTD% -f tmp                             && REM trivial compression case, creates tmp.zst
+%ZSTD% -df tmp.zst                        && REM trivial decompression case (overwrites tmp)
+echo test : too large compression level (must fail)
+%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1)
+echo test : compress to stdout
+%ZSTD% tmp -c > tmpCompressed                 
+%ZSTD% tmp --stdout > tmpCompressed       && REM long command format
+echo test : null-length file roundtrip
+echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1)
+echo test : decompress file with wrong suffix (must fail)
+%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1)
+%ZSTD% -d tmpCompressed -c > tmpResult    && REM decompression using stdout   
+%ZSTD% --decompress tmpCompressed -c > tmpResult
+%ZSTD% --decompress tmpCompressed --stdout > tmpResult
+REM %ZSTD% -d    < tmp.zst > NUL              && REM combine decompression, stdin & stdout
+REM %ZSTD% -d  - < tmp.zst > NUL
+%ZSTD% -dc   < tmp.zst > NUL
+%ZSTD% -dc - < tmp.zst > NUL
+%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1)
+%ZSTD% -q -f tmp
+%ZSTD% -q --force tmp
+%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1)
+cp tmp tmp2.zst
+%ZSTD% -df tmp2.zst && (echo should have failed : wrong format && exit /b 1)
+rm tmp2.zst
+
+echo. && echo **** frame concatenation ****
+
+echo hello > hello.tmp
+echo world! > world.tmp
+cat hello.tmp world.tmp > helloworld.tmp
+%ZSTD% -c hello.tmp > hello.zstd
+%ZSTD% -c world.tmp > world.zstd
+cat hello.zstd world.zstd > helloworld.zstd
+%ZSTD% -dc helloworld.zstd > result.tmp
+cat result.tmp
+fc /b helloworld.tmp result.tmp
+rm *.tmp *.zstd
+
+echo frame concatenation test completed
+
+
+REM echo. && echo **** flush write error test ****
+
+REM echo echo foo ^| %ZSTD% ^> v:\full
+REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1)
+REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full"
+REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1)
+
+
+echo. && echo **** dictionary tests ****
+
+%DATAGEN% > tmpDict
+%DATAGEN% -g1M | md5sum > tmp1
+%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2
+fc tmp1 tmp2
+%ZSTD% --train *.c *.h -o tmpDict
+%ZSTD% xxhash.c -D tmpDict -of tmp
+%ZSTD% -d tmp -D tmpDict -of result
+fc xxhash.c result
+
+
+echo. && echo **** multiple files tests ****
+
+%DATAGEN% -s1        > tmp1 2> NUL
+%DATAGEN% -s2 -g100K > tmp2 2> NUL
+%DATAGEN% -s3 -g1M   > tmp3 2> NUL
+%ZSTD% -f tmp*
+echo compress tmp* :
+ls -ls tmp*
+rm tmp1 tmp2 tmp3
+echo decompress tmp* :
+%ZSTD% -df *.zst
+ls -ls tmp*
+
+echo compress tmp* into stdout ^> tmpall :
+%ZSTD% -c tmp1 tmp2 tmp3 > tmpall
+ls -ls tmp*
+echo decompress tmpall* into stdout ^> tmpdec :
+cp tmpall tmpall2
+%ZSTD% -dc tmpall* > tmpdec
+ls -ls tmp*
+echo compress multiple files including a missing one (notHere) :
+%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1)
+
+
+echo. && echo **** integrity tests ****
+echo test one file (tmp1.zst)
+%ZSTD% -t tmp1.zst
+%ZSTD% --test tmp1.zst
+echo test multiple files (*.zst)
+%ZSTD% -t *.zst
+echo test good and bad files (*)
+%ZSTD% -t * && (echo bad files not detected! && exit /b 1)
+
+
+echo. && echo **** zstd round-trip tests ****
+
+CALL %ROUNDTRIPTEST%
+CALL %ROUNDTRIPTEST% -g15K       && REM TableID==3
+CALL %ROUNDTRIPTEST% -g127K      && REM TableID==2
+CALL %ROUNDTRIPTEST% -g255K      && REM TableID==1
+CALL %ROUNDTRIPTEST% -g513K      && REM TableID==0
+CALL %ROUNDTRIPTEST% -g512K 6    && REM greedy, hash chain
+CALL %ROUNDTRIPTEST% -g512K 16   && REM btlazy2 
+CALL %ROUNDTRIPTEST% -g512K 19   && REM btopt
+
+rm tmp*
+echo Param = %1
+if NOT "%1"=="--test-large-data" (
+    echo skipping large data tests
+    exit /b 0
+)
+
+CALL %ROUNDTRIPTEST% -g270000000 1
+CALL %ROUNDTRIPTEST% -g270000000 2
+CALL %ROUNDTRIPTEST% -g270000000 3
+
+CALL %ROUNDTRIPTEST% -g140000000 -P60 4
+CALL %ROUNDTRIPTEST% -g140000000 -P60 5
+CALL %ROUNDTRIPTEST% -g140000000 -P60 6
+
+CALL %ROUNDTRIPTEST% -g70000000 -P70 7
+CALL %ROUNDTRIPTEST% -g70000000 -P70 8
+CALL %ROUNDTRIPTEST% -g70000000 -P70 9
+
+CALL %ROUNDTRIPTEST% -g35000000 -P75 10
+CALL %ROUNDTRIPTEST% -g35000000 -P75 11
+CALL %ROUNDTRIPTEST% -g35000000 -P75 12
+
+CALL %ROUNDTRIPTEST% -g18000000 -P80 13
+CALL %ROUNDTRIPTEST% -g18000000 -P80 14
+CALL %ROUNDTRIPTEST% -g18000000 -P80 15
+CALL %ROUNDTRIPTEST% -g18000000 -P80 16
+CALL %ROUNDTRIPTEST% -g18000000 -P80 17
+
+CALL %ROUNDTRIPTEST% -g50000000 -P94 18
+CALL %ROUNDTRIPTEST% -g50000000 -P94 19
+
+CALL %ROUNDTRIPTEST% -g99000000 -P99 20
+CALL %ROUNDTRIPTEST% -g6000000000 -P99 1
+
+rm tmp*
+exit /b 0
--- a/programs/tests/playTests.sh
+++ b/programs/tests/playTests.sh
--- a/programs/tests/playTestsMSYS.sh
+++ b/programs/tests/playTestsMSYS.sh
@ -0,0 +1,160 @@
+#!/bin/sh -e
+
+die() {
+    echo "$@" 1>&2
+    exit 1
+}
+
+roundTripTest() {
+    if [ -n "$3" ]; then
+        local c="$3"
+        local p="$2"
+    else
+        local c="$2"
+    fi
+
+    rm -f tmp1 tmp2
+    echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
+    ./datagen $1 $p | md5sum > tmp1
+    ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d  | md5sum > tmp2
+    fc tmp1 tmp2
+}
+
+[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
+
+
+echo -e "\n**** simple tests **** "
+./datagen > tmp
+$ZSTD -f tmp                             # trivial compression case, creates tmp.zst
+$ZSTD -df tmp.zst                        # trivial decompression case (overwrites tmp)
+echo "test : too large compression level (must fail)"
+$ZSTD -99 tmp && die "too large compression level undetected"
+echo "test : compress to stdout"
+$ZSTD tmp -c > tmpCompressed                 
+$ZSTD tmp --stdout > tmpCompressed       # long command format
+echo "test : null-length file roundtrip"
+echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
+echo "test : decompress file with wrong suffix (must fail)"
+$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
+$ZSTD -d tmpCompressed -c > tmpResult    # decompression using stdout   
+$ZSTD --decompress tmpCompressed -c > tmpResult
+$ZSTD --decompress tmpCompressed --stdout > tmpResult
+#$ZSTD -d    < tmp.zst > /dev/null        # combine decompression, stdin & stdout
+#$ZSTD -d  - < tmp.zst > /dev/null
+$ZSTD -dc   < tmp.zst > /dev/null
+$ZSTD -dc - < tmp.zst > /dev/null
+$ZSTD -q tmp && die "overwrite check failed!"
+$ZSTD -q -f tmp
+$ZSTD -q --force tmp
+$ZSTD -df tmp && die "should have refused : wrong extension"
+cp tmp tmp2.zst
+$ZSTD -df tmp2.zst && die "should have failed : wrong format"
+rm tmp2.zst
+
+echo -e "\n**** frame concatenation **** "
+
+echo "hello " > hello.tmp
+echo "world!" > world.tmp
+cat hello.tmp world.tmp > helloworld.tmp
+$ZSTD -c hello.tmp > hello.zstd
+$ZSTD -c world.tmp > world.zstd
+cat hello.zstd world.zstd > helloworld.zstd
+$ZSTD -dc helloworld.zstd > result.tmp
+cat result.tmp
+fc helloworld.tmp result.tmp
+rm ./*.tmp ./*.zstd
+
+echo frame concatenation test completed
+
+
+echo -e "\n**** dictionary tests **** "
+
+./datagen > tmpDict
+./datagen -g1M | md5sum > tmp1
+./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
+fc tmp1 tmp2
+$ZSTD --train *.c *.h -o tmpDict
+$ZSTD xxhash.c -D tmpDict -of tmp
+$ZSTD -d tmp -D tmpDict -of result
+fc xxhash.c result
+
+
+echo -e "\n**** multiple files tests **** "
+
+./datagen -s1        > tmp1 2> /dev/null
+./datagen -s2 -g100K > tmp2 2> /dev/null
+./datagen -s3 -g1M   > tmp3 2> /dev/null
+$ZSTD -f tmp*
+echo "compress tmp* : "
+ls -ls tmp*
+rm tmp1 tmp2 tmp3
+echo "decompress tmp* : "
+$ZSTD -df *.zst
+ls -ls tmp*
+echo "compress tmp* into stdout > tmpall : "
+$ZSTD -c tmp1 tmp2 tmp3 > tmpall
+ls -ls tmp*
+echo "decompress tmpall* into stdout > tmpdec : "
+cp tmpall tmpall2
+$ZSTD -dc tmpall* > tmpdec
+ls -ls tmp*
+echo "compress multiple files including a missing one (notHere) : "
+$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
+
+echo -e "\n**** integrity tests **** "
+echo "test one file (tmp1.zst) "
+$ZSTD -t tmp1.zst
+$ZSTD --test tmp1.zst
+echo "test multiple files (*.zst) "
+$ZSTD -t *.zst
+echo "test good and bad files (*) "
+$ZSTD -t * && die "bad files not detected !"
+
+echo -e "\n**** zstd round-trip tests **** "
+
+roundTripTest
+roundTripTest -g15K       # TableID==3
+roundTripTest -g127K      # TableID==2
+roundTripTest -g255K      # TableID==1
+roundTripTest -g513K      # TableID==0
+roundTripTest -g512K 6    # greedy, hash chain
+roundTripTest -g512K 16   # btlazy2 
+roundTripTest -g512K 19   # btopt
+
+rm tmp*
+
+if [ "$1" != "--test-large-data" ]; then
+    echo "Skipping large data tests"
+    exit 0
+fi
+
+roundTripTest -g270000000 1
+roundTripTest -g270000000 2
+roundTripTest -g270000000 3
+
+roundTripTest -g140000000 -P60 4
+roundTripTest -g140000000 -P60 5
+roundTripTest -g140000000 -P60 6
+
+roundTripTest -g70000000 -P70 7
+roundTripTest -g70000000 -P70 8
+roundTripTest -g70000000 -P70 9
+
+roundTripTest -g35000000 -P75 10
+roundTripTest -g35000000 -P75 11
+roundTripTest -g35000000 -P75 12
+
+roundTripTest -g18000000 -P80 13
+roundTripTest -g18000000 -P80 14
+roundTripTest -g18000000 -P80 15
+roundTripTest -g18000000 -P80 16
+roundTripTest -g18000000 -P80 17
+
+roundTripTest -g50000000 -P94 18
+roundTripTest -g50000000 -P94 19
+
+roundTripTest -g99000000 -P99 20
+roundTripTest -g6000000000 -P99 1
+
+rm tmp*
+
--- a/programs/tests/roundTripTest.bat
+++ b/programs/tests/roundTripTest.bat
@ -0,0 +1,8 @@
+@echo off
+if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) 
+rm -f tmp1 tmp2
+echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d
+%DATAGEN% %1 %P% | md5sum > tmp1
+%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d  | md5sum > tmp2
+fc tmp1 tmp2
+EXIT /B %ERRORLEVEL%
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@ -49,14 +49,11 @@
 *  OS-specific Includes
 **************************************/
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include <fcntl.h>    /* _O_BINARY */
-#  include <io.h>       /* _setmode, _isatty */
-#  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+#  include <io.h>       /* _isatty */
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
 #else
   extern int fileno(FILE *stream);  /* triggers fileno() within <stdio.h> on POSIX */
 #  include <unistd.h>   /* isatty */
-#  define SET_BINARY_MODE(file)
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
 #endif

--- a/projects/VS2010/datagen/datagen.vcxproj
+++ b/projects/VS2010/datagen/datagen.vcxproj
@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{037E781E-81A6-494B-B1B3-438AB1200523}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>datagen</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IntDir>$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IntDir>$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <EnablePREfast>false</EnablePREfast>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\programs\datagencli.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\programs\datagen.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
--- a/projects/VS2010/datagen/datagen.vcxproj.filters
+++ b/projects/VS2010/datagen/datagen.vcxproj.filters
@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\programs\datagen.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\programs\datagencli.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\programs\datagen.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>
--- a/projects/VS2010/zstd.sln
+++ b/projects/VS2010/zstd.sln
@ -9,6 +9,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\full
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Win32 = Debug|Win32
@ -49,6 +51,14 @@ Global
 		{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32
 		{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64
 		{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.ActiveCfg = Debug|Win32
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.Build.0 = Debug|Win32
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.ActiveCfg = Debug|x64
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.Build.0 = Debug|x64
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.ActiveCfg = Release|Win32
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.Build.0 = Release|Win32
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.ActiveCfg = Release|x64
+		{037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE