From 56f1f0e3ddbd4078a78a11be148ca89dc1cc51fb Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 26 Sep 2017 11:21:36 -0700
Subject: [PATCH 1/8] write summary for --list on multiple files

---
 programs/fileio.c | 106 +++++++++++++++++++++++++++++++---------------
 1 file changed, 71 insertions(+), 35 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 623c4f4d..db325e44 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -347,13 +347,16 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
     fileHandle = fopen(fileName, "rb");
     if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
     fileSize = UTIL_getFileSize(fileName);
-    if (fileSize > DICTSIZE_MAX)
+    if (fileSize > DICTSIZE_MAX) {
         EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
                         fileName, DICTSIZE_MAX >> 20);   /* avoid extreme cases */
+    }
     *bufferPtr = malloc((size_t)fileSize);
     if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
-    { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
-      if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
+    {   size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
+        if (readSize!=fileSize)
+            EXM_THROW(35, "Error reading dictionary file %s", fileName);
+    }
     fclose(fileHandle);
     return (size_t)fileSize;
 }
@@ -970,7 +973,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     char*  dstFileName = (char*)malloc(FNSPACE);
     size_t const suffixSize = suffix ? strlen(suffix) : 0;
     U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
-    int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
+    int const isRegularFile = (nbFiles > 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);  /* won't write frame content size when nbFiles > 1 */
     cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
 
     /* init */
@@ -1700,10 +1703,11 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
 typedef struct {
     int numActualFrames;
     int numSkippableFrames;
-    unsigned long long decompressedSize;
+    U64 decompressedSize;
     int decompUnavailable;
-    unsigned long long compressedSize;
+    U64 compressedSize;
     int usesCheck;
+    U32 nbFiles;
 } fileInfo_t;
 
 /** getFileInfo() :
@@ -1720,14 +1724,16 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
         DISPLAY("Error: could not open source file %s\n", inFileName);
         return 3;
     }
-    info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
+    info->compressedSize = UTIL_getFileSize(inFileName);
 
     /* begin analyzing frame */
     for ( ; ; ) {
         BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
         size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
         if (numBytesRead < ZSTD_frameHeaderSize_min) {
-            if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
+            if ( feof(srcFile)
+              && (numBytesRead == 0)
+              && (info->compressedSize > 0) ) {
                 break;
             }
             else if (feof(srcFile)) {
@@ -1829,6 +1835,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
         }
     }  /* end analyzing frame */
     fclose(srcFile);
+    info->nbFiles = 1;
     return detectError;
 }
 
@@ -1841,16 +1848,17 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
     const char* const checkString = (info->usesCheck ? "XXH64" : "None");
     if (displayLevel <= 2) {
         if (!info->decompUnavailable) {
-            DISPLAYOUT("Skippable  Non-Skippable  Compressed  Uncompressed  Ratio  Check  Filename\n");
-            DISPLAYOUT("%9d  %13d  %7.2f %2s  %9.2f %2s  %5.3f  %5s  %s\n",
-                    info->numSkippableFrames, info->numActualFrames,
+            DISPLAYOUT("%6d  %5d  %7.2f %2s  %9.2f %2s  %5.3f  %5s  %s\n",
+                    info->numSkippableFrames + info->numActualFrames,
+                    info->numSkippableFrames,
                     compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
                     ratio, checkString, inFileName);
         } else {
-            DISPLAYOUT("Skippable  Non-Skippable  Compressed  Check  Filename\n");
-            DISPLAYOUT("%9d  %13d  %7.2f MB  %5s  %s\n",
-                    info->numSkippableFrames, info->numActualFrames,
-                    compressedSizeUnit, checkString, inFileName);
+            DISPLAYOUT("%6d  %5d  %7.2f %2s                       %5s  %s\n",
+                    info->numSkippableFrames + info->numActualFrames,
+                    info->numSkippableFrames,
+                    compressedSizeUnit, unitStr,
+                    checkString, inFileName);
         }
     } else {
         DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
@@ -1867,33 +1875,40 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
     }
 }
 
+static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
+{
+    fileInfo_t total;
+    total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames;
+    total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames;
+    total.compressedSize = fi1.compressedSize + fi2.compressedSize;
+    total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize;
+    total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable;
+    total.usesCheck = fi1.usesCheck & fi2.usesCheck;
+    total.nbFiles = fi1.nbFiles + fi2.nbFiles;
+    return total;
+}
 
-static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
+static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
     /* initialize info to avoid warnings */
     fileInfo_t info;
     memset(&info, 0, sizeof(info));
-    DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
-    {
-        int const error = getFileInfo(&info, inFileName);
+    {   int const error = getFileInfo(&info, inFileName);
         if (error == 1) {
             /* display error, but provide output */
-            DISPLAY("An error occurred with getting file info\n");
+            DISPLAY("An error occurred while getting file info \n");
         }
         else if (error == 2) {
-            DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
-            if (displayLevel > 2) {
-                DISPLAYOUT("\n");
-            }
+            DISPLAYOUT("File %s not compressed by zstd \n", inFileName);
+            if (displayLevel > 2) DISPLAYOUT("\n");
             return 1;
         }
         else if (error == 3) {
-            /* error occurred with opening the file */
-            if (displayLevel > 2) {
-                DISPLAYOUT("\n");
-            }
+            /* error occurred while opening the file */
+            if (displayLevel > 2) DISPLAYOUT("\n");
             return 1;
         }
         displayInfo(inFileName, &info, displayLevel);
+        *total = FIO_addFInfo(*total, info);
         return error;
     }
 }
@@ -1903,15 +1918,36 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
         DISPLAYOUT("No files given\n");
         return 0;
     }
-    DISPLAYOUT("===========================================\n");
-    DISPLAYOUT("Printing information about compressed files\n");
-    DISPLAYOUT("===========================================\n");
-    DISPLAYOUT("Number of files listed: %u\n", numFiles);
-    {
-        int error = 0;
+    DISPLAYOUT("Frames  Skips  Compressed  Uncompressed  Ratio  Check  Filename\n");
+    {   int error = 0;
         unsigned u;
+        fileInfo_t total;
+        memset(&total, 0, sizeof(total));
+        total.usesCheck = 1;
         for (u=0; u<numFiles;u++) {
-            error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
+            error |= FIO_listFile(&total, filenameTable[u], displayLevel);
+        }
+        if (numFiles > 1) {
+            unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
+            const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
+            double const compressedSizeUnit = (double)total.compressedSize / unit;
+            double const decompressedSizeUnit = (double)total.decompressedSize / unit;
+            double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize;
+            const char* const checkString = (total.usesCheck ? "XXH64" : "");
+            DISPLAYOUT("----------------------------------------------------------------- \n");
+            if (total.decompUnavailable) {
+                DISPLAYOUT("%6d  %5d  %7.2f %2s                       %5s  %u files\n",
+                        total.numSkippableFrames + total.numActualFrames,
+                        total.numSkippableFrames,
+                        compressedSizeUnit, unitStr,
+                        checkString, total.nbFiles);
+            } else {
+                DISPLAYOUT("%6d  %5d  %7.2f %2s  %9.2f %2s  %5.3f  %5s  %u files\n",
+                        total.numSkippableFrames + total.numActualFrames,
+                        total.numSkippableFrames,
+                        compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
+                        ratio, checkString, total.nbFiles);
+            }
         }
         return error;
     }

From 3095ca8c56864552d910564cfd4f0b18f56f9b81 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 26 Sep 2017 13:53:50 -0700
Subject: [PATCH 2/8] fixed minor conversion warnings for g++ on Linux

U64 is not considered equivalent to unsigned long long
---
 programs/fileio.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index db325e44..3ca35e1a 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1864,10 +1864,12 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
         DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
         DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
         DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
-                    compressedSizeUnit, unitStr, info->compressedSize);
+                    compressedSizeUnit, unitStr,
+                    (unsigned long long)info->compressedSize);
         if (!info->decompUnavailable) {
             DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
-                    decompressedSizeUnit, unitStr, info->decompressedSize);
+                    decompressedSizeUnit, unitStr,
+                    (unsigned long long)info->decompressedSize);
             DISPLAYOUT("Ratio: %.4f\n", ratio);
         }
         DISPLAYOUT("Check: %s\n", checkString);

From c233bdbaeef497a946256aeb6aff9fc108d4d2d7 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 22 Sep 2017 14:04:39 -0700
Subject: [PATCH 3/8] Increase maximum window size

* Maximum window size in 32-bit mode is 1GB, since allocations for 2GB fail
  on my Mac.
* Maximum window size in 64-bit mode is 2GB, since that is the largest
  power of 2 that works with the overflow prevention.
* Allow `--long=windowLog` to set the window log, along with
  `--zstd=wlog=#`. These options also set the window size during
  decompression, but don't override `--memory=#` if it is set.
* Present a helpful error message when the window size is too large during
  decompression.
* The long range matcher defaults to a hash log 7 less than the window log,
  which keeps it at 20 for window log 27.
* Keep the default long range matcher window size and the default maximum
  window size at 27 for the API and CLI.
* Add tests that use the maximum window size and hash size for compression
  and decompression.
---
 lib/common/zstd_internal.h       |  1 +
 lib/compress/zstd_compress.c     | 14 ++++++-------
 lib/compress/zstd_ldm.c          |  8 +++++--
 lib/compress/zstd_ldm.h          |  2 +-
 lib/decompress/zstd_decompress.c |  6 +++---
 lib/zstd.h                       | 10 +++++----
 programs/fileio.c                | 35 +++++++++++++++++++++++++++++--
 programs/zstd.1                  | 18 ++++++++++++----
 programs/zstd.1.md               | 20 ++++++++++++------
 programs/zstdcli.c               | 27 ++++++++++++++++++++++--
 tests/playTests.sh               | 36 ++++++++++++++++++++++++--------
 tests/zstreamtest.c              |  2 ++
 12 files changed, 139 insertions(+), 40 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 403c0cbd..1defee67 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -102,6 +102,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 #define BIT0   1
 
 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 782439f7..e84149bd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -429,7 +429,7 @@ size_t ZSTD_CCtxParam_setParameter(
     case ZSTD_p_enableLongDistanceMatching :
         if (value != 0) {
             ZSTD_cLevelToCCtxParams(params);
-            params->cParams.windowLog = ZSTD_LDM_WINDOW_LOG;
+            params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
         }
         return ZSTD_ldm_initializeParameters(&params->ldmParams, value);
 
@@ -1599,7 +1599,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
     const BYTE* ip = (const BYTE*)src;
     BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
-    U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog;
+    U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
 
     if (cctx->appliedParams.fParams.checksumFlag && srcSize)
         XXH64_update(&cctx->xxhState, src, srcSize);
@@ -1630,9 +1630,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
          *    windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
          */
         if (cctx->lowLimit > (3U<<29)) {
-            U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
+            U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
             U32 const current = (U32)(ip - cctx->base);
-            U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog);
+            U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog);
             U32 const correction = current - newCurrent;
             ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
             ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
@@ -1688,7 +1688,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     U32   const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536);   /* 0-3 */
     U32   const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength;   /* 0-3 */
     U32   const checksumFlag = params.fParams.checksumFlag>0;
-    U32   const windowSize = 1U << params.cParams.windowLog;
+    U32   const windowSize = (U32)1 << params.cParams.windowLog;
     U32   const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
     BYTE  const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
     U32   const fcsCode = params.fParams.contentSizeFlag ?
@@ -1788,7 +1788,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
     ZSTD_compressionParameters const cParams =
             ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0);
-    return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
+    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
 }
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -1837,7 +1837,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
     case ZSTD_btopt:
     case ZSTD_btultra:
         if (srcSize >= HASH_READ_SIZE)
-            ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
+            ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
         break;
 
     default:
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index e40007c1..be50872c 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -14,14 +14,14 @@
 
 #define LDM_BUCKET_SIZE_LOG 3
 #define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LOG 20
+#define LDM_HASH_RLOG 7
 #define LDM_HASH_CHAR_OFFSET 10
 
 size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
 {
     ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
     params->enableLdm = enableLdm>0;
-    params->hashLog = LDM_HASH_LOG;
+    params->hashLog = 0;
     params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
     params->minMatchLength = LDM_MIN_MATCH_LENGTH;
     params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET;
@@ -30,6 +30,10 @@ size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
 
 void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog)
 {
+    if (params->hashLog == 0) {
+        params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG);
+        assert(params->hashLog <= ZSTD_HASHLOG_MAX);
+    }
     if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) {
         params->hashEveryLog =
                 windowLog < params->hashLog ? 0 : windowLog - params->hashLog;
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 7a624839..d6d3d42c 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -20,7 +20,7 @@ extern "C" {
 *  Long distance matching
 ***************************************/
 
-#define ZSTD_LDM_WINDOW_LOG 27
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
 #define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999
 
 /** ZSTD_compressBlock_ldm_generic() :
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index c91e082e..ebfa93c3 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -35,7 +35,7 @@
 *  Frames requiring more memory will be rejected.
 */
 #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-#  define ZSTD_MAXWINDOWSIZE_DEFAULT (((U64)1 << ZSTD_WINDOWLOG_MAX) + 1)   /* defined within zstd.h */
+#  define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
 #endif
 
 
@@ -913,7 +913,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l
             offset = 0;
         else {
             ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
-            ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
+            ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
             assert(ofBits <= MaxOff);
             if (MEM_32bits() && longOffsets) {
                 U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
@@ -1159,7 +1159,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
             offset = 0;
         else {
             ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
-            ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
+            ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
             assert(ofBits <= MaxOff);
             if (MEM_32bits() && longOffsets) {
                 U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
diff --git a/lib/zstd.h b/lib/zstd.h
index a6266dfd..b7c75d5f 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -376,8 +376,8 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
 #define ZSTD_MAGIC_DICTIONARY       0xEC30A437   /* v0.7+ */
 
-#define ZSTD_WINDOWLOG_MAX_32  27
-#define ZSTD_WINDOWLOG_MAX_64  27
+#define ZSTD_WINDOWLOG_MAX_32  30
+#define ZSTD_WINDOWLOG_MAX_64  31
 #define ZSTD_WINDOWLOG_MAX    ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
 #define ZSTD_WINDOWLOG_MIN     10
 #define ZSTD_HASHLOG_MAX       MIN(ZSTD_WINDOWLOG_MAX, 30)
@@ -941,7 +941,9 @@ typedef enum {
                               * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * Special: value 0 means "do not change windowLog". */
+                              * Special: value 0 means "do not change windowLog".
+                              * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
+                              * requires setting the maximum window size at least as large during decompression. */
     ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
                               * Resulting table size is (1 << (hashLog+2)).
                               * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -1009,7 +1011,7 @@ typedef enum {
                           * Larger values increase memory usage and compression ratio, but decrease
                           * compression speed.
                           * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
-                          * (default: 20). */
+                          * (default: windowlog - 7). */
     ZSTD_p_ldmMinMatch,  /* Minimum size of searched matches for long distance matcher.
                           * Larger/too small values usually decrease compression ratio.
                           * Must be clamped between ZSTD_LDM_MINMATCH_MIN
diff --git a/programs/fileio.c b/programs/fileio.c
index 623c4f4d..a4b50fc2 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -37,6 +37,7 @@
 #  include <io.h>
 #endif
 
+#include "bitstream.h"
 #include "mem.h"
 #include "fileio.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
@@ -1167,6 +1168,35 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
     return 0;
 }
 
+static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
+{
+    ZSTD_frameHeader header;
+    /* No special help for these errors */
+    if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
+        return;
+    /* Try to decode the frame header */
+    ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
+    if (ret == 0) {
+        U32 const windowSize = (U32)header.windowSize;
+        U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
+        U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
+        assert(header.windowSize <= (U64)((U32)-1));
+        assert(g_memLimit > 0);
+        DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
+                        srcFileName, header.windowSize, g_memLimit);
+        if (windowLog <= ZSTD_WINDOWLOG_MAX) {
+            DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
+                            srcFileName, windowLog, windowMB);
+            return;
+        }
+    } else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
+        DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
+                        srcFileName, ZSTD_getErrorName(ret));
+        return;
+    }
+    DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
+                    srcFileName, ZSTD_WINDOWLOG_MAX);
+}
 
 /** FIO_decompressFrame() :
  *  @return : size of decoded zstd frame, or an error code
@@ -1183,8 +1213,8 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
     ZSTD_resetDStream(ress->dctx);
     if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20;   /* display last 20 characters */
 
-    /* Header loading (optional, saves one loop) */
-    {   size_t const toRead = 9;
+    /* Header loading : ensures ZSTD_getFrameHeader() will succeed */
+    {   size_t const toRead = ZSTD_FRAMEHEADERSIZE_MAX;
         if (ress->srcBufferLoaded < toRead)
             ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
     }
@@ -1197,6 +1227,7 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
         if (ZSTD_isError(readSizeHint)) {
             DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
                             srcFileName, ZSTD_getErrorName(readSizeHint));
+            FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
             return FIO_ERROR_FRAME_DECODING;
         }
 
diff --git a/programs/zstd.1 b/programs/zstd.1
index 0fad1d27..b68899b2 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -104,8 +104,11 @@ Display information related to a zstd compressed file, such as size, ratio, and
 unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
 .
 .TP
-\fB\-\-long\fR
-enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
+\fB\-\-long[=#]\fR
+enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
+.
+.IP
+Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
 .
 .TP
 \fB\-T#\fR, \fB\-\-threads=#\fR
@@ -190,6 +193,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\.
 Limit dictionary to specified size (default: 112640)\.
 .
 .TP
+\fB\-B#\fR
+Split input files in blocks of size # (default: no split)
+.
+.TP
 \fB\-\-dictID=#\fR
 A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
 .
@@ -267,7 +274,10 @@ There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fas
 Specify the maximum number of bits for a match distance\.
 .
 .IP
-The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
+The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
+.
+.IP
+Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
 .
 .TP
 \fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
@@ -340,7 +350,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo
 The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
 .
 .TP
-\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
+\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
 Specify the minimum searched length of a match for long distance matching\.
 .
 .IP
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index e446422b..562f8721 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -105,12 +105,16 @@ the last one takes effect.
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
     Note that decompression will also require more memory when using these levels.
-* `--long`:
-    enables long distance matching.
+* `--long[=#]`:
+    enables long distance matching with `#` `windowLog`, if not `#` is not
+    present it defaults to `27`.
     This increases the window size (`windowLog`) and memory usage for both the
-    compressor and decompressor. This setting is designed to improve the
-    compression ratio for files with long matches at a large distance
-    (up to the maximum window size, 128 MiB).
+    compressor and decompressor.
+    This setting is designed to improve the compression ratio for files with
+    long matches at a large distance.
+
+    Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
+    `--memory=windowSize` needs to be passed to the decompressor.
 * `-T#`, `--threads=#`:
     Compress using `#` threads (default: 1).
     If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -275,7 +279,11 @@ The list of available _options_:
     The higher number of increases the chance to find a match which usually
     improves compression ratio.
     It also increases memory requirements for the compressor and decompressor.
-    The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
+    The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
+    platforms and 31 (2 GiB) on 64-bit platforms.
+
+    Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
+    `--memory=windowSize` needs to be passed to the decompressor.
 
 - `hashLog`=_hlog_, `hlog`=_hlog_:
     Specify the maximum number of bits for a hash table.
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 89e97c29..3478028c 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -72,6 +72,7 @@ static const char*    g_defaultDictName = "dictionary";
 static const unsigned g_defaultMaxDictSize = 110 KB;
 static const int      g_defaultDictCLevel = 3;
 static const unsigned g_defaultSelectivityLevel = 9;
+static const unsigned g_defaultMaxWindowLog = 27;
 #define OVERLAP_LOG_DEFAULT 9999
 #define LDM_PARAM_DEFAULT 9999  /* Default for parameters where 0 is valid */
 static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
@@ -129,7 +130,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -l     : print information about zstd compressed files \n");
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAY( "--long  : enable long distance matching (requires more memory)\n");
+    DISPLAY( "--long[=#]  : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
 #ifdef ZSTD_MULTITHREAD
     DISPLAY( " -T#    : use # threads for compression (default:1) \n");
     DISPLAY( " -B#    : select size of each job (default:0==automatic) \n");
@@ -459,7 +460,6 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
                     if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
                     if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
-                    if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
                     if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
                     if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
                     if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
@@ -514,6 +514,22 @@ int main(int argCount, const char* argv[])
                     if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
+                    if (longCommandWArg(&argument, "--long")) {
+                        unsigned ldmWindowLog = 0;
+                        ldmFlag = 1;
+                        /* Parse optional window log */
+                        if (*argument == '=') {
+                            ++argument;
+                            ldmWindowLog = readU32FromChar(&argument);
+                        } else if (*argument != 0) {
+                            /* Invalid character following --long */
+                            CLEAN_RETURN(badusage(programName));
+                        }
+                        /* Only set windowLog if not already set by --zstd */
+                        if (compressionParams.windowLog == 0)
+                            compressionParams.windowLog = ldmWindowLog;
+                        continue;
+                    }
                     /* fall-through, will trigger bad_usage() later on */
                 }
 
@@ -830,6 +846,13 @@ int main(int argCount, const char* argv[])
 #endif
     } else {  /* decompression or test */
 #ifndef ZSTD_NODECOMPRESS
+        if (memLimit == 0) {
+            if (compressionParams.windowLog == 0)
+                memLimit = (U32)1 << g_defaultMaxWindowLog;
+            else {
+                memLimit = (U32)1 << (compressionParams.windowLog & 31);
+            }
+        }
         FIO_setMemLimit(memLimit);
         if (filenameIdx==1 && outFileName)
             operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 38b7a196..b3e0b8ab 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -13,11 +13,16 @@ roundTripTest() {
         cLevel="$2"
         proba=""
     fi
+    if [ -n "$4" ]; then
+        dLevel="$4"
+    else
+        dLevel="$cLevel"
+    fi
 
     rm -f tmp1 tmp2
-    $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
+    $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
     ./datagen $1 $proba | $MD5SUM > tmp1
-    ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d  | $MD5SUM > tmp2
+    ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel  | $MD5SUM > tmp2
     $DIFF -q tmp1 tmp2
 }
 
@@ -29,12 +34,17 @@ fileRoundTripTest() {
         local_c="$2"
         local_p=""
     fi
+    if [ -n "$4" ]; then
+        local_d="$4"
+    else
+        local_d="$local_c"
+    fi
 
     rm -f tmp.zstd tmp.md5.1 tmp.md5.2
-    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
+    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
     ./datagen $1 $local_p > tmp
     cat tmp | $MD5SUM > tmp.md5.1
-    $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
+    $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
     $DIFF -q tmp.md5.1 tmp.md5.2
 }
 
@@ -669,16 +679,24 @@ roundTripTest -g140000000 -P60 "5 --long"
 roundTripTest -g70000000 -P70 "8 --long"
 roundTripTest -g18000001 -P80  "18 --long"
 fileRoundTripTest -g4100M -P99 "1 --long"
+# Test large window logs
+roundTripTest -g4100M -P50 "1 --long=30"
+roundTripTest -g4100M -P50 "1 --long --zstd=wlog=30,clog=30"
+# Test parameter parsing
+roundTripTest -g1M -P50 "1 --long=30" " --memory=1024MB"
+roundTripTest -g1M -P50 "1 --long=30 --zstd=wlog=29" " --memory=512MB"
+roundTripTest -g1M -P50 "1 --long=30" " --long=29 --memory=1024MB"
+roundTripTest -g1M -P50 "1 --long=30" " --zstd=wlog=29 --memory=1024MB"
 
 
 if [ -n "$hasMT" ]
 then
     $ECHO "\n**** zstdmt long round-trip tests **** "
-    roundTripTest -g99000000 -P99 "20 -T2"
-    roundTripTest -g6000000000 -P99 "1 -T2"
-    roundTripTest -g1500000000 -P97 "1 -T999"
-    fileRoundTripTest -g4195M -P98 " -T0"
-    roundTripTest -g1500000000 -P97 "1 --long -T999"
+    roundTripTest -g99000000 -P99 "20 -T2" " "
+    roundTripTest -g6000000000 -P99 "1 -T2" " "
+    roundTripTest -g1500000000 -P97 "1 -T999" " "
+    fileRoundTripTest -g4195M -P98 " -T0" " "
+    roundTripTest -g1500000000 -P97 "1 --long=23 -T2" " "
 else
     $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
 fi
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e52335da..3f190f05 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -50,6 +50,7 @@ static const U32 g_cLevelMax_smallTests = 10;
 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
 static const U32 prime32 = 2654435761U;
+static const U32 windowLogMax = 27;
 
 
 /*-************************************
@@ -1380,6 +1381,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
 
                 /* mess with compression parameters */
                 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;

From 471aa385b3f236fdab550ce5cdfbfa5e3aa9f9d5 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 26 Sep 2017 14:03:43 -0700
Subject: [PATCH 4/8] [fuzz] Speed up round trip tests

* Enforce smaller maximum values for parameters
* Adjust parameters to the source size

The memory usage is reduced by about 5x, which makes the fuzzers run at
least twice as fast, even more so with ASAN/MSAN enabled.
---
 tests/fuzz/block_round_trip.c  | 29 ++++++---------
 tests/fuzz/simple_round_trip.c | 14 +++----
 tests/fuzz/stream_round_trip.c |  9 +++--
 tests/fuzz/zstd_helpers.c      | 68 ++++++++++++++++++++++++++--------
 tests/fuzz/zstd_helpers.h      |  6 ++-
 5 files changed, 80 insertions(+), 46 deletions(-)

diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c
index 3b3f2ff6..64ca5fc4 100644
--- a/tests/fuzz/block_round_trip.c
+++ b/tests/fuzz/block_round_trip.c
@@ -34,27 +34,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
                             const void *src, size_t srcSize)
 {
-  int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
-  size_t ret = ZSTD_compressBegin(cctx, cLevel);
+    int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
+    ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0);
+    size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize);
+    FUZZ_ZASSERT(ret);
 
-  if (ZSTD_isError(ret)) {
-      fprintf(stderr, "ZSTD_compressBegin() error: %s\n",
-              ZSTD_getErrorName(ret));
-      return ret;
-  }
-
-  ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize);
-  if (ZSTD_isError(ret)) {
-    fprintf(stderr, "ZSTD_compressBlock() error: %s\n", ZSTD_getErrorName(ret));
-    return ret;
-  }
-  if (ret == 0) {
+    ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize);
+    FUZZ_ZASSERT(ret);
+    if (ret == 0) {
         FUZZ_ASSERT(resultCapacity >= srcSize);
         memcpy(result, src, srcSize);
         return srcSize;
-  }
-  ZSTD_decompressBegin(dctx);
-  return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret);
+    }
+    ZSTD_decompressBegin(dctx);
+    return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret);
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
@@ -87,7 +80,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     {
         size_t const result =
             roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
-        FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result));
+        FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 617e45df..0921106d 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -41,21 +41,17 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
         size_t err;
 
         ZSTD_CCtx_reset(cctx);
-        FUZZ_setRandomParameters(cctx, &seed);
+        FUZZ_setRandomParameters(cctx, srcSize, &seed);
         err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
-        if (err != 0) {
-            return err;
-        }
+        FUZZ_ZASSERT(err);
+        FUZZ_ASSERT(err == 0);
         cSize = out.pos;
     } else {
         int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
         cSize = ZSTD_compressCCtx(
             cctx, compressed, compressedCapacity, src, srcSize, cLevel);
     }
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
-        return cSize;
-    }
+    FUZZ_ZASSERT(cSize);
     return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
 }
 
@@ -87,7 +83,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     {
         size_t const result =
             roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
-        FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result));
+        FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index e3fdd3b8..72d70495 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -57,7 +57,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
 {
     size_t dstSize = 0;
     ZSTD_CCtx_reset(cctx);
-    FUZZ_setRandomParameters(cctx, &seed);
+    FUZZ_setRandomParameters(cctx, srcSize, &seed);
 
     while (srcSize > 0) {
         ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
@@ -85,7 +85,10 @@ static size_t compress(uint8_t *dst, size_t capacity,
                     /* Reset the compressor when the frame is finished */
                     if (ret == 0) {
                         ZSTD_CCtx_reset(cctx);
-                        FUZZ_setRandomParameters(cctx, &seed);
+                        if ((FUZZ_rand(&seed) & 7) == 0) {
+                            size_t const remaining = in.size - in.pos;
+                            FUZZ_setRandomParameters(cctx, remaining, &seed);
+                        }
                         mode = -1;
                     }
                     break;
@@ -146,7 +149,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         size_t const cSize = compress(cBuf, neededBufSize, src, size);
         size_t const rSize =
             ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize);
-        FUZZ_ASSERT_MSG(!ZSTD_isError(rSize), ZSTD_getErrorName(rSize));
+        FUZZ_ZASSERT(rSize);
         FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index c5bef027..60289847 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -13,30 +13,68 @@
 #include "fuzz_helpers.h"
 #include "zstd.h"
 
-static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
-                    unsigned max, uint32_t *state) {
-  unsigned const value = FUZZ_rand32(state, min, max);
-  FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
+static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value)
+{
+    FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
 }
 
-void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state)
+static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
+                    unsigned max, uint32_t *state) {
+    unsigned const value = FUZZ_rand32(state, min, max);
+    set(cctx, param, value);
+}
+
+ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
 {
-    setRand(cctx, ZSTD_p_windowLog, ZSTD_WINDOWLOG_MIN, 23, state);
-    setRand(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN, 23, state);
-    setRand(cctx, ZSTD_p_chainLog, ZSTD_CHAINLOG_MIN, 24, state);
-    setRand(cctx, ZSTD_p_searchLog, ZSTD_SEARCHLOG_MIN, 9, state);
-    setRand(cctx, ZSTD_p_minMatch, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX,
-            state);
-    setRand(cctx, ZSTD_p_targetLength, ZSTD_TARGETLENGTH_MIN,
-            ZSTD_TARGETLENGTH_MAX, state);
-    setRand(cctx, ZSTD_p_compressionStrategy, ZSTD_fast, ZSTD_btultra, state);
+    /* Select compression parameters */
+    ZSTD_compressionParameters cParams;
+    cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15);
+    cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
+    cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
+    cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
+    cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
+                                              ZSTD_SEARCHLENGTH_MAX);
+    cParams.targetLength = FUZZ_rand32(state, ZSTD_TARGETLENGTH_MIN,
+                                              ZSTD_TARGETLENGTH_MAX);
+    cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
+    return ZSTD_adjustCParams(cParams, srcSize, 0);
+}
+
+ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state)
+{
+    /* Select frame parameters */
+    ZSTD_frameParameters fParams;
+    fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1);
+    fParams.checksumFlag = FUZZ_rand32(state, 0, 1);
+    fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1);
+    return fParams;
+}
+
+ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
+{
+    ZSTD_parameters params;
+    params.cParams = FUZZ_randomCParams(srcSize, state);
+    params.fParams = FUZZ_randomFParams(state);
+    return params;
+}
+
+void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
+{
+    ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
+    set(cctx, ZSTD_p_windowLog, cParams.windowLog);
+    set(cctx, ZSTD_p_hashLog, cParams.hashLog);
+    set(cctx, ZSTD_p_chainLog, cParams.chainLog);
+    set(cctx, ZSTD_p_searchLog, cParams.searchLog);
+    set(cctx, ZSTD_p_minMatch, cParams.searchLength);
+    set(cctx, ZSTD_p_targetLength, cParams.targetLength);
+    set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
     /* Select frame parameters */
     setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
     setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
     setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
     /* Select long distance matchig parameters */
     setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
-    setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 24, state);
+    setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
     setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
             ZSTD_LDM_MINMATCH_MAX, state);
     setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h
index c2e3388a..3856bebe 100644
--- a/tests/fuzz/zstd_helpers.h
+++ b/tests/fuzz/zstd_helpers.h
@@ -21,7 +21,11 @@
 extern "C" {
 #endif
 
-void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state);
+void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state);
+
+ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state);
+ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state);
+ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state);
 
 
 #ifdef __cplusplus

From 6c41adfb28dc3b8497d401ec8994b305eb563cc0 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 27 Sep 2017 11:16:24 -0700
Subject: [PATCH 5/8] [libzstd] pthread function prefixed with ZSTD_

* `sed -i 's/pthread_/ZSTD_pthread_/g' lib/{,common,compress,decompress,dictBuilder}/*.[hc]`
* Fix up `lib/common/threading.[hc]`
* `sed -i s/PTHREAD_MUTEX_LOCK/ZSTD_PTHREAD_MUTEX_LOCK/g lib/compress/zstdmt_compress.c`
---
 lib/common/pool.c              | 60 +++++++++++-----------
 lib/common/threading.c         | 12 ++---
 lib/common/threading.h         | 72 +++++++++++++++-----------
 lib/compress/zstdmt_compress.c | 92 +++++++++++++++++-----------------
 lib/dictBuilder/cover.c        | 28 +++++------
 5 files changed, 140 insertions(+), 124 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index 5f19f331..9e6f802e 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -33,7 +33,7 @@ typedef struct POOL_job_s {
 struct POOL_ctx_s {
     ZSTD_customMem customMem;
     /* Keep track of the threads */
-    pthread_t *threads;
+    ZSTD_pthread_t *threads;
     size_t numThreads;
 
     /* The queue is a circular buffer */
@@ -48,11 +48,11 @@ struct POOL_ctx_s {
     int queueEmpty;
 
     /* The mutex protects the queue */
-    pthread_mutex_t queueMutex;
+    ZSTD_pthread_mutex_t queueMutex;
     /* Condition variable for pushers to wait on when the queue is full */
-    pthread_cond_t queuePushCond;
+    ZSTD_pthread_cond_t queuePushCond;
     /* Condition variables for poppers to wait on when the queue is empty */
-    pthread_cond_t queuePopCond;
+    ZSTD_pthread_cond_t queuePopCond;
     /* Indicates if the queue is shutting down */
     int shutdown;
 };
@@ -67,14 +67,14 @@ static void* POOL_thread(void* opaque) {
     if (!ctx) { return NULL; }
     for (;;) {
         /* Lock the mutex and wait for a non-empty queue or until shutdown */
-        pthread_mutex_lock(&ctx->queueMutex);
+        ZSTD_pthread_mutex_lock(&ctx->queueMutex);
 
         while (ctx->queueEmpty && !ctx->shutdown) {
-            pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+            ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
         }
         /* empty => shutting down: so stop */
         if (ctx->queueEmpty) {
-            pthread_mutex_unlock(&ctx->queueMutex);
+            ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
             return opaque;
         }
         /* Pop a job off the queue */
@@ -83,17 +83,17 @@ static void* POOL_thread(void* opaque) {
             ctx->numThreadsBusy++;
             ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
             /* Unlock the mutex, signal a pusher, and run the job */
-            pthread_mutex_unlock(&ctx->queueMutex);
-            pthread_cond_signal(&ctx->queuePushCond);
+            ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+            ZSTD_pthread_cond_signal(&ctx->queuePushCond);
 
             job.function(job.opaque);
 
             /* If the intended queue size was 0, signal after finishing job */
             if (ctx->queueSize == 1) {
-                pthread_mutex_lock(&ctx->queueMutex);
+                ZSTD_pthread_mutex_lock(&ctx->queueMutex);
                 ctx->numThreadsBusy--;
-                pthread_mutex_unlock(&ctx->queueMutex);
-                pthread_cond_signal(&ctx->queuePushCond);
+                ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+                ZSTD_pthread_cond_signal(&ctx->queuePushCond);
         }   }
     }  /* for (;;) */
     /* Unreachable */
@@ -120,12 +120,12 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
     ctx->queueTail = 0;
     ctx->numThreadsBusy = 0;
     ctx->queueEmpty = 1;
-    (void)pthread_mutex_init(&ctx->queueMutex, NULL);
-    (void)pthread_cond_init(&ctx->queuePushCond, NULL);
-    (void)pthread_cond_init(&ctx->queuePopCond, NULL);
+    (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+    (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+    (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
-    ctx->threads = (pthread_t*)ZSTD_malloc(numThreads * sizeof(pthread_t), customMem);
+    ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
     ctx->numThreads = 0;
     ctx->customMem = customMem;
     /* Check for errors */
@@ -133,7 +133,7 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
     /* Initialize the threads */
     {   size_t i;
         for (i = 0; i < numThreads; ++i) {
-            if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+            if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
                 ctx->numThreads = i;
                 POOL_free(ctx);
                 return NULL;
@@ -148,25 +148,25 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
 */
 static void POOL_join(POOL_ctx *ctx) {
     /* Shut down the queue */
-    pthread_mutex_lock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
     ctx->shutdown = 1;
-    pthread_mutex_unlock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
     /* Wake up sleeping threads */
-    pthread_cond_broadcast(&ctx->queuePushCond);
-    pthread_cond_broadcast(&ctx->queuePopCond);
+    ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
+    ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
     /* Join all of the threads */
     {   size_t i;
         for (i = 0; i < ctx->numThreads; ++i) {
-            pthread_join(ctx->threads[i], NULL);
+            ZSTD_pthread_join(ctx->threads[i], NULL);
     }   }
 }
 
 void POOL_free(POOL_ctx *ctx) {
     if (!ctx) { return; }
     POOL_join(ctx);
-    pthread_mutex_destroy(&ctx->queueMutex);
-    pthread_cond_destroy(&ctx->queuePushCond);
-    pthread_cond_destroy(&ctx->queuePopCond);
+    ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
+    ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
+    ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
     ZSTD_free(ctx->queue, ctx->customMem);
     ZSTD_free(ctx->threads, ctx->customMem);
     ZSTD_free(ctx, ctx->customMem);
@@ -176,7 +176,7 @@ size_t POOL_sizeof(POOL_ctx *ctx) {
     if (ctx==NULL) return 0;  /* supports sizeof NULL */
     return sizeof(*ctx)
         + ctx->queueSize * sizeof(POOL_job)
-        + ctx->numThreads * sizeof(pthread_t);
+        + ctx->numThreads * sizeof(ZSTD_pthread_t);
 }
 
 /**
@@ -198,12 +198,12 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
     POOL_ctx* const ctx = (POOL_ctx*)ctxVoid;
     if (!ctx) { return; }
 
-    pthread_mutex_lock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
     {   POOL_job const job = {function, opaque};
 
         /* Wait until there is space in the queue for the new job */
         while (isQueueFull(ctx) && !ctx->shutdown) {
-          pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+          ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
         }
         /* The queue is still going => there is space */
         if (!ctx->shutdown) {
@@ -212,8 +212,8 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
             ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
         }
     }
-    pthread_mutex_unlock(&ctx->queueMutex);
-    pthread_cond_signal(&ctx->queuePopCond);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+    ZSTD_pthread_cond_signal(&ctx->queuePopCond);
 }
 
 #else  /* ZSTD_MULTITHREAD  not defined */
diff --git a/lib/common/threading.c b/lib/common/threading.c
index a82c975b..8be8c8da 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -35,12 +35,12 @@ int g_ZSTD_threading_useles_symbol;
 
 static unsigned __stdcall worker(void *arg)
 {
-    pthread_t* const thread = (pthread_t*) arg;
+    ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
     thread->arg = thread->start_routine(thread->arg);
     return 0;
 }
 
-int pthread_create(pthread_t* thread, const void* unused,
+int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
     (void)unused;
@@ -54,16 +54,16 @@ int pthread_create(pthread_t* thread, const void* unused,
         return 0;
 }
 
-int _pthread_join(pthread_t * thread, void **value_ptr)
+int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 {
     DWORD result;
 
-    if (!thread->handle) return 0;
+    if (!thread.handle) return 0;
 
-    result = WaitForSingleObject(thread->handle, INFINITE);
+    result = WaitForSingleObject(thread.handle, INFINITE);
     switch (result) {
     case WAIT_OBJECT_0:
-        if (value_ptr) *value_ptr = thread->arg;
+        if (value_ptr) *value_ptr = thread.arg;
         return 0;
     case WAIT_ABANDONED:
         return EINVAL;
diff --git a/lib/common/threading.h b/lib/common/threading.h
index 8194bc6f..197770db 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -44,32 +44,31 @@ extern "C" {
 
 
 /* mutex */
-#define pthread_mutex_t           CRITICAL_SECTION
-#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), 0)
-#define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
-#define pthread_mutex_lock(a)     EnterCriticalSection((a))
-#define pthread_mutex_unlock(a)   LeaveCriticalSection((a))
+#define ZSTD_pthread_mutex_t           CRITICAL_SECTION
+#define ZSTD_pthread_mutex_init(a, b)  (InitializeCriticalSection((a)), 0)
+#define ZSTD_pthread_mutex_destroy(a)  DeleteCriticalSection((a))
+#define ZSTD_pthread_mutex_lock(a)     EnterCriticalSection((a))
+#define ZSTD_pthread_mutex_unlock(a)   LeaveCriticalSection((a))
 
 /* condition variable */
-#define pthread_cond_t             CONDITION_VARIABLE
-#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
-#define pthread_cond_destroy(a)    /* No delete */
-#define pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
-#define pthread_cond_signal(a)     WakeConditionVariable((a))
-#define pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
+#define ZSTD_pthread_cond_t             CONDITION_VARIABLE
+#define ZSTD_pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
+#define ZSTD_pthread_cond_destroy(a)    /* No delete */
+#define ZSTD_pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
+#define ZSTD_pthread_cond_signal(a)     WakeConditionVariable((a))
+#define ZSTD_pthread_cond_broadcast(a)  WakeAllConditionVariable((a))
 
-/* pthread_create() and pthread_join() */
+/* ZSTD_pthread_create() and ZSTD_pthread_join() */
 typedef struct {
     HANDLE handle;
     void* (*start_routine)(void*);
     void* arg;
-} pthread_t;
+} ZSTD_pthread_t;
 
-int pthread_create(pthread_t* thread, const void* unused,
+int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
                    void* (*start_routine) (void*), void* arg);
 
-#define pthread_join(a, b) _pthread_join(&(a), (b))
-int _pthread_join(pthread_t* thread, void** value_ptr);
+int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
 
 /**
  * add here more wrappers as required
@@ -80,23 +79,40 @@ int _pthread_join(pthread_t* thread, void** value_ptr);
 /* ===   POSIX Systems   === */
 #  include <pthread.h>
 
+#define ZSTD_pthread_mutex_t            pthread_mutex_t
+#define ZSTD_pthread_mutex_init(a, b)   pthread_mutex_init((a), (b))
+#define ZSTD_pthread_mutex_destroy(a)   pthread_mutex_destroy((a))
+#define ZSTD_pthread_mutex_lock(a)      pthread_mutex_lock((a))
+#define ZSTD_pthread_mutex_unlock(a)    pthread_mutex_unlock((a))
+
+#define ZSTD_pthread_cond_t             pthread_cond_t
+#define ZSTD_pthread_cond_init(a, b)    pthread_cond_init((a), (b))
+#define ZSTD_pthread_cond_destroy(a)    pthread_cond_destroy((a))
+#define ZSTD_pthread_cond_wait(a, b)    pthread_cond_wait((a), (b))
+#define ZSTD_pthread_cond_signal(a)     pthread_cond_signal((a))
+#define ZSTD_pthread_cond_broadcast(a)  pthread_cond_broadcast((a))
+
+#define ZSTD_pthread_t                  pthread_t
+#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
+#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+
 #else  /* ZSTD_MULTITHREAD not defined */
 /* No multithreading support */
 
-#define pthread_mutex_t int   /* #define rather than typedef, because sometimes pthread support is implicit, resulting in duplicated symbols */
-#define pthread_mutex_init(a,b)    ((void)a, 0)
-#define pthread_mutex_destroy(a)
-#define pthread_mutex_lock(a)
-#define pthread_mutex_unlock(a)
+typedef int ZSTD_pthread_mutex_t;
+#define ZSTD_pthread_mutex_init(a, b)   ((void)a, 0)
+#define ZSTD_pthread_mutex_destroy(a)
+#define ZSTD_pthread_mutex_lock(a)
+#define ZSTD_pthread_mutex_unlock(a)
 
-#define pthread_cond_t int
-#define pthread_cond_init(a,b)     ((void)a, 0)
-#define pthread_cond_destroy(a)
-#define pthread_cond_wait(a,b)
-#define pthread_cond_signal(a)
-#define pthread_cond_broadcast(a)
+typedef int ZSTD_pthread_cond_t;
+#define ZSTD_pthread_cond_init(a, b)    ((void)a, 0)
+#define ZSTD_pthread_cond_destroy(a)
+#define ZSTD_pthread_cond_wait(a, b)
+#define ZSTD_pthread_cond_signal(a)
+#define ZSTD_pthread_cond_broadcast(a)
 
-/* do not use pthread_t */
+/* do not use ZSTD_pthread_t */
 
 #endif /* ZSTD_MULTITHREAD */
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index ecb799ab..3bdb98e2 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -53,22 +53,22 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
 }
 
 #define MUTEX_WAIT_TIME_DLEVEL 6
-#define PTHREAD_MUTEX_LOCK(mutex) {               \
+#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) {               \
     if (ZSTD_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) {   \
         unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
-        pthread_mutex_lock(mutex);                \
+        ZSTD_pthread_mutex_lock(mutex);                \
         {   unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
             unsigned long long const elapsedTime = (afterTime-beforeTime); \
             if (elapsedTime > 1000) {  /* or whatever threshold you like; I'm using 1 millisecond here */ \
                 DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
                    elapsedTime, #mutex);          \
         }   }                                     \
-    } else pthread_mutex_lock(mutex);             \
+    } else ZSTD_pthread_mutex_lock(mutex);             \
 }
 
 #else
 
-#  define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
+#  define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m)
 #  define DEBUG_PRINTHEX(l,p,n) {}
 
 #endif
@@ -85,7 +85,7 @@ typedef struct buffer_s {
 static const buffer_t g_nullBuffer = { NULL, 0 };
 
 typedef struct ZSTDMT_bufferPool_s {
-    pthread_mutex_t poolMutex;
+    ZSTD_pthread_mutex_t poolMutex;
     size_t bufferSize;
     unsigned totalBuffers;
     unsigned nbBuffers;
@@ -99,7 +99,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
     ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
         sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
     if (bufPool==NULL) return NULL;
-    if (pthread_mutex_init(&bufPool->poolMutex, NULL)) {
+    if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) {
         ZSTD_free(bufPool, cMem);
         return NULL;
     }
@@ -116,7 +116,7 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
     if (!bufPool) return;   /* compatibility with free on NULL */
     for (u=0; u<bufPool->totalBuffers; u++)
         ZSTD_free(bufPool->bTable[u].start, bufPool->cMem);
-    pthread_mutex_destroy(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_destroy(&bufPool->poolMutex);
     ZSTD_free(bufPool, bufPool->cMem);
 }
 
@@ -127,10 +127,10 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
                             + (bufPool->totalBuffers - 1) * sizeof(buffer_t);
     unsigned u;
     size_t totalBufferSize = 0;
-    pthread_mutex_lock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
     for (u=0; u<bufPool->totalBuffers; u++)
         totalBufferSize += bufPool->bTable[u].size;
-    pthread_mutex_unlock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
 
     return poolSize + totalBufferSize;
 }
@@ -146,20 +146,20 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
 {
     size_t const bSize = bufPool->bufferSize;
     DEBUGLOG(5, "ZSTDMT_getBuffer");
-    pthread_mutex_lock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
     if (bufPool->nbBuffers) {   /* try to use an existing buffer */
         buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)];
         size_t const availBufferSize = buf.size;
         if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) {
             /* large enough, but not too much */
-            pthread_mutex_unlock(&bufPool->poolMutex);
+            ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
             return buf;
         }
         /* size conditions not respected : scratch this buffer, create new one */
         DEBUGLOG(5, "existing buffer does not meet size conditions => freeing");
         ZSTD_free(buf.start, bufPool->cMem);
     }
-    pthread_mutex_unlock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
     /* create new buffer */
     DEBUGLOG(5, "create a new buffer");
     {   buffer_t buffer;
@@ -175,13 +175,13 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 {
     if (buf.start == NULL) return;   /* compatible with release on NULL */
     DEBUGLOG(5, "ZSTDMT_releaseBuffer");
-    pthread_mutex_lock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
     if (bufPool->nbBuffers < bufPool->totalBuffers) {
         bufPool->bTable[bufPool->nbBuffers++] = buf;  /* stored for later use */
-        pthread_mutex_unlock(&bufPool->poolMutex);
+        ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
         return;
     }
-    pthread_mutex_unlock(&bufPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
     /* Reached bufferPool capacity (should not happen) */
     DEBUGLOG(5, "buffer pool capacity reached => freeing ");
     ZSTD_free(buf.start, bufPool->cMem);
@@ -206,7 +206,7 @@ static ZSTD_CCtx_params ZSTDMT_makeJobCCtxParams(ZSTD_CCtx_params const params)
 /* a single CCtx Pool can be invoked from multiple threads in parallel */
 
 typedef struct {
-    pthread_mutex_t poolMutex;
+    ZSTD_pthread_mutex_t poolMutex;
     unsigned totalCCtx;
     unsigned availCCtx;
     ZSTD_customMem cMem;
@@ -219,7 +219,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
     unsigned u;
     for (u=0; u<pool->totalCCtx; u++)
         ZSTD_freeCCtx(pool->cctx[u]);  /* note : compatible with free on NULL */
-    pthread_mutex_destroy(&pool->poolMutex);
+    ZSTD_pthread_mutex_destroy(&pool->poolMutex);
     ZSTD_free(pool, pool->cMem);
 }
 
@@ -231,7 +231,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
         sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
     if (!cctxPool) return NULL;
-    if (pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
+    if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
         ZSTD_free(cctxPool, cMem);
         return NULL;
     }
@@ -247,7 +247,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
 /* only works during initialization phase, not during compression */
 static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
 {
-    pthread_mutex_lock(&cctxPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
     {   unsigned const nbThreads = cctxPool->totalCCtx;
         size_t const poolSize = sizeof(*cctxPool)
                                 + (nbThreads-1)*sizeof(ZSTD_CCtx*);
@@ -256,7 +256,7 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
         for (u=0; u<nbThreads; u++) {
             totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
         }
-        pthread_mutex_unlock(&cctxPool->poolMutex);
+        ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
         return poolSize + totalCCtxSize;
     }
 }
@@ -264,14 +264,14 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
 static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
 {
     DEBUGLOG(5, "ZSTDMT_getCCtx");
-    pthread_mutex_lock(&cctxPool->poolMutex);
+    ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
     if (cctxPool->availCCtx) {
         cctxPool->availCCtx--;
         {   ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx];
-            pthread_mutex_unlock(&cctxPool->poolMutex);
+            ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
             return cctx;
     }   }
-    pthread_mutex_unlock(&cctxPool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
     DEBUGLOG(5, "create one more CCtx");
     return ZSTD_createCCtx_advanced(cctxPool->cMem);   /* note : can be NULL, when creation fails ! */
 }
@@ -279,7 +279,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
 static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return;   /* compatibility with release on NULL */
-    pthread_mutex_lock(&pool->poolMutex);
+    ZSTD_pthread_mutex_lock(&pool->poolMutex);
     if (pool->availCCtx < pool->totalCCtx)
         pool->cctx[pool->availCCtx++] = cctx;
     else {
@@ -287,7 +287,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
         DEBUGLOG(5, "CCtx pool overflow : free cctx");
         ZSTD_freeCCtx(cctx);
     }
-    pthread_mutex_unlock(&pool->poolMutex);
+    ZSTD_pthread_mutex_unlock(&pool->poolMutex);
 }
 
 
@@ -305,8 +305,8 @@ typedef struct {
     unsigned lastChunk;
     unsigned jobCompleted;
     unsigned jobScanned;
-    pthread_mutex_t* jobCompleted_mutex;
-    pthread_cond_t* jobCompleted_cond;
+    ZSTD_pthread_mutex_t* jobCompleted_mutex;
+    ZSTD_pthread_cond_t* jobCompleted_cond;
     ZSTD_CCtx_params params;
     const ZSTD_CDict* cdict;
     ZSTDMT_CCtxPool* cctxPool;
@@ -373,11 +373,11 @@ _endJob:
     ZSTDMT_releaseCCtx(job->cctxPool, cctx);
     ZSTDMT_releaseBuffer(job->bufPool, job->src);
     job->src = g_nullBuffer; job->srcStart = NULL;
-    PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
+    ZSTD_PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
     job->jobCompleted = 1;
     job->jobScanned = 0;
-    pthread_cond_signal(job->jobCompleted_cond);
-    pthread_mutex_unlock(job->jobCompleted_mutex);
+    ZSTD_pthread_cond_signal(job->jobCompleted_cond);
+    ZSTD_pthread_mutex_unlock(job->jobCompleted_mutex);
 }
 
 
@@ -395,8 +395,8 @@ struct ZSTDMT_CCtx_s {
     ZSTDMT_jobDescription* jobs;
     ZSTDMT_bufferPool* bufPool;
     ZSTDMT_CCtxPool* cctxPool;
-    pthread_mutex_t jobCompleted_mutex;
-    pthread_cond_t jobCompleted_cond;
+    ZSTD_pthread_mutex_t jobCompleted_mutex;
+    ZSTD_pthread_cond_t jobCompleted_cond;
     size_t targetSectionSize;
     size_t inBuffSize;
     size_t dictSize;
@@ -459,11 +459,11 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
         ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
-    if (pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
+    if (ZSTD_pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
         ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
-    if (pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
+    if (ZSTD_pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
         ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
@@ -503,8 +503,8 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     ZSTD_free(mtctx->jobs, mtctx->cMem);
     ZSTDMT_freeCCtxPool(mtctx->cctxPool);
     ZSTD_freeCDict(mtctx->cdictLocal);
-    pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
-    pthread_cond_destroy(&mtctx->jobCompleted_cond);
+    ZSTD_pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
+    ZSTD_pthread_cond_destroy(&mtctx->jobCompleted_cond);
     ZSTD_free(mtctx, mtctx->cMem);
     return 0;
 }
@@ -649,12 +649,12 @@ static size_t ZSTDMT_compress_advanced_internal(
         unsigned chunkID;
         for (chunkID=0; chunkID<nbChunks; chunkID++) {
             DEBUGLOG(5, "waiting for chunk %u ", chunkID);
-            PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
+            ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
             while (mtctx->jobs[chunkID].jobCompleted==0) {
                 DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID);
-                pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
+                ZSTD_pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
             }
-            pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
+            ZSTD_pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
             DEBUGLOG(5, "ready to write chunk %u ", chunkID);
 
             mtctx->jobs[chunkID].srcStart = NULL;
@@ -729,12 +729,12 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
     DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted");
     while (zcs->doneJobID < zcs->nextJobID) {
         unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
-        PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+        ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
         while (zcs->jobs[jobID].jobCompleted==0) {
             DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID);   /* we want to block when waiting for data to flush */
-            pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
+            ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
         }
-        pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+        ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex);
         zcs->doneJobID++;
     }
 }
@@ -923,13 +923,13 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
 {
     unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
     if (zcs->doneJobID == zcs->nextJobID) return 0;   /* all flushed ! */
-    PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+    ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
     while (zcs->jobs[wJobID].jobCompleted==0) {
         DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
-        if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; }  /* nothing ready to be flushed => skip */
-        pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);  /* block when nothing available to flush */
+        if (!blockToFlush) { ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; }  /* nothing ready to be flushed => skip */
+        ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);  /* block when nothing available to flush */
     }
-    pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+    ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex);
     /* compression job completed : output can be flushed */
     {   ZSTDMT_jobDescription job = zcs->jobs[wJobID];
         if (!job.jobScanned) {
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index f6500b3d..efdffddb 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -711,8 +711,8 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
  * compiled with multithreaded support.
  */
 typedef struct COVER_best_s {
-  pthread_mutex_t mutex;
-  pthread_cond_t cond;
+  ZSTD_pthread_mutex_t mutex;
+  ZSTD_pthread_cond_t cond;
   size_t liveJobs;
   void *dict;
   size_t dictSize;
@@ -725,8 +725,8 @@ typedef struct COVER_best_s {
  */
 static void COVER_best_init(COVER_best_t *best) {
   if (best==NULL) return; /* compatible with init on NULL */
-  (void)pthread_mutex_init(&best->mutex, NULL);
-  (void)pthread_cond_init(&best->cond, NULL);
+  (void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
+  (void)ZSTD_pthread_cond_init(&best->cond, NULL);
   best->liveJobs = 0;
   best->dict = NULL;
   best->dictSize = 0;
@@ -741,11 +741,11 @@ static void COVER_best_wait(COVER_best_t *best) {
   if (!best) {
     return;
   }
-  pthread_mutex_lock(&best->mutex);
+  ZSTD_pthread_mutex_lock(&best->mutex);
   while (best->liveJobs != 0) {
-    pthread_cond_wait(&best->cond, &best->mutex);
+    ZSTD_pthread_cond_wait(&best->cond, &best->mutex);
   }
-  pthread_mutex_unlock(&best->mutex);
+  ZSTD_pthread_mutex_unlock(&best->mutex);
 }
 
 /**
@@ -759,8 +759,8 @@ static void COVER_best_destroy(COVER_best_t *best) {
   if (best->dict) {
     free(best->dict);
   }
-  pthread_mutex_destroy(&best->mutex);
-  pthread_cond_destroy(&best->cond);
+  ZSTD_pthread_mutex_destroy(&best->mutex);
+  ZSTD_pthread_cond_destroy(&best->cond);
 }
 
 /**
@@ -771,9 +771,9 @@ static void COVER_best_start(COVER_best_t *best) {
   if (!best) {
     return;
   }
-  pthread_mutex_lock(&best->mutex);
+  ZSTD_pthread_mutex_lock(&best->mutex);
   ++best->liveJobs;
-  pthread_mutex_unlock(&best->mutex);
+  ZSTD_pthread_mutex_unlock(&best->mutex);
 }
 
 /**
@@ -789,7 +789,7 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
   }
   {
     size_t liveJobs;
-    pthread_mutex_lock(&best->mutex);
+    ZSTD_pthread_mutex_lock(&best->mutex);
     --best->liveJobs;
     liveJobs = best->liveJobs;
     /* If the new dictionary is better */
@@ -812,9 +812,9 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
       best->parameters = parameters;
       best->compressedSize = compressedSize;
     }
-    pthread_mutex_unlock(&best->mutex);
+    ZSTD_pthread_mutex_unlock(&best->mutex);
     if (liveJobs == 0) {
-      pthread_cond_broadcast(&best->cond);
+      ZSTD_pthread_cond_broadcast(&best->cond);
     }
   }
 }

From 763f8b5e453c2add84541f750a7ca5ad3664bb33 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 27 Sep 2017 12:09:52 -0700
Subject: [PATCH 6/8] Change c++ test to use CXX and CXXFLAGS environment
 variables

Fix OS-X warning on compiling .c files with clang++
Also changed test name from gpptest to cpptest, since it's no longer g++ specific
---
 Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 7baff751..b09da547 100644
--- a/Makefile
+++ b/Makefile
@@ -112,7 +112,7 @@ CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_B
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
 
-.PHONY: install clangtest gpptest armtest usan asan uasan
+.PHONY: install clangtest armtest usan asan uasan
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
@@ -179,8 +179,10 @@ ppcfuzz: clean
 ppc64fuzz: clean
 	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
-gpptest: clean
-	CC=$(CXX) $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+.PHONY: cpptest
+cpptest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cpptest: clean
+	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
 
 gcc5test: clean
 	gcc-5 -v

From 60ca44b545b98ace8d4e84dce1fa387a49f75524 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 27 Sep 2017 12:24:13 -0700
Subject: [PATCH 7/8] switched name to cxxtest

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index b09da547..5b3f5fdf 100644
--- a/Makefile
+++ b/Makefile
@@ -179,9 +179,9 @@ ppcfuzz: clean
 ppc64fuzz: clean
 	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
-.PHONY: cpptest
-cpptest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
-cpptest: clean
+.PHONY: cxxtest
+cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cxxtest: clean
 	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
 
 gcc5test: clean

From b555b7ef412c6497fa1e806aa91652e852c740aa Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 27 Sep 2017 15:29:07 -0700
Subject: [PATCH 8/8] [libzstd][opt] Simplify repcode logic

---
 lib/compress/zstd_opt.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index fd102da2..c47ce23a 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -529,7 +529,14 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
            } else {
                 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
                 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
-                opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
+                /* If opt[cur].off == ZSTD_REP_MOVE_OPT, then mlen != 1.
+                 * offset ZSTD_REP_MOVE_OPT is used for the special case
+                 * litLength == 0, where offset 0 means something special.
+                 * mlen == 1 means the previous byte was stored as a literal,
+                 * so they are mutually exclusive.
+                 */
+                assert(!(opt[cur].off == ZSTD_REP_MOVE_OPT && mlen == 1));
+                opt[cur].rep[0] = (opt[cur].off == ZSTD_REP_MOVE_OPT) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
            }
 
             best_mlen = minMatch;
@@ -804,7 +811,8 @@ size_t ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             } else {
                 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
                 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
-                opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
+                assert(!(opt[cur].off == ZSTD_REP_MOVE_OPT && mlen == 1));
+                opt[cur].rep[0] = (opt[cur].off == ZSTD_REP_MOVE_OPT) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
             }
 
             best_mlen = minMatch;