Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

.travis.yml

@@ -12,7 +12,7 @@ matrix:
       os: linux
       sudo: false
 
-    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
+    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
       os: linux
       sudo: false
       language: cpp
@@ -69,7 +69,7 @@ matrix:
 
 
     # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
+    - env: Ubu=14.04 Cmd="make armtest"
       dist: trusty
       sudo: required
       addons:
@@ -78,7 +78,17 @@ matrix:
             - qemu-system-arm
             - qemu-user-static
             - gcc-arm-linux-gnueabi
-            - libc6-dev-armel-cross 
+            - libc6-dev-armel-cross
+
+    # Ubuntu 14.04 LTS Server Edition 64 bit
+    - env: Ubu=14.04 Cmd="make aarch64test"
+      dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-arm
+            - qemu-user-static
             - gcc-aarch64-linux-gnu
             - libc6-dev-arm64-cross
 
@@ -91,6 +101,7 @@ matrix:
             - qemu-system-ppc
             - qemu-user-static
             - gcc-powerpc-linux-gnu
+            - libc6-dev-armel-cross
 
     - env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
       os: linux
@@ -101,7 +112,10 @@ matrix:
           packages:
             - valgrind
 
-    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
+    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean
+                       && make c99test && make clean && make gnu99test && make clean
+                       && make clangtest && make clean && make -C contrib/pzstd googletest32 
+                       && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
       os: linux
       dist: trusty
       sudo: required
@@ -127,7 +141,7 @@ matrix:
             - libc6-dev-i386
             - gcc-multilib
 
-    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
+    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll"
       os: linux
       dist: trusty
       sudo: required

NEWS

@@ -1,6 +1,10 @@
+v1.1.3
+cli : new : commands for advanced parameters, by Przemyslaw Skibinski
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+
 v1.1.2
-API : streaming : decompression : changed : implicit reset on starting new frames without init
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
-API : experimental : added : dictID retrieval functions
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
 API : zbuff : changed : prototypes now generate deprecation warnings
 lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
 lib : changed : only public ZSTD_ symbols are now exposed

appveyor.yml

@@ -63,7 +63,8 @@ build_script:
       make -v &&
       cc -v &&
       ECHO make %MAKE_PARAMS% &&
-      make %MAKE_PARAMS%
+      make %MAKE_PARAMS% &&
+      make -C tests test-symbols
     )
   - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
       COPY programs\zstd.exe bin\zstd.exe &&

build/VS2005/zstdlib/zstdlib.vcproj

@@ -359,6 +359,10 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
 				>

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -29,6 +29,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />

build/VS2010/libzstd/libzstd.vcxproj

@@ -29,6 +29,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />

build/cmake/lib/CMakeLists.txt

@@ -69,6 +69,7 @@ SET(Sources
         ${LIBRARY_DIR}/decompress/zstd_decompress.c
         ${LIBRARY_DIR}/dictBuilder/divsufsort.c
         ${LIBRARY_DIR}/dictBuilder/zdict.c
+        ${LIBRARY_DIR}/deprecated/zbuff_common.c
         ${LIBRARY_DIR}/deprecated/zbuff_compress.c
         ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
 

doc/zstd_manual.html

@@ -88,29 +88,21 @@
     note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
 </p></pre><BR>
 
-<h3>Helper functions</h3><pre></pre><b><pre>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
+<h3>Helper functions</h3><pre><b>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
 size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
 
-<h3>Compression context</h3><pre>   When compressing many times,
-   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
-   This will make workload friendlier for system's memory.
-   Use one context per thread for parallel execution in multi-threaded environments. 
-</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
-ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-</pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 </b><p>    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 </p></pre><BR>
 
-<h3>Decompression context</h3><pre></pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+<h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-</pre></b><BR>
+</b></pre><BR>
 <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 </b><p>   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 </p></pre><BR>
@@ -200,20 +192,20 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
   Start a new compression by initializing ZSTD_CStream.
   Use ZSTD_initCStream() to start a new compression operation.
-  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+  Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
 
   Use ZSTD_compressStream() repetitively to consume input stream.
   The function will automatically update both `pos` fields.
   Note that it may not consume the entire input, in which case `pos < size`,
   and it's up to the caller to present again remaining data.
   @return : a size hint, preferred nb of bytes to use as input for next function call
-           (it's just a hint, to help latency a little, any other value will work fine)
-           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
             or an error code, which can be tested using ZSTD_isError().
+            Note 1 : it's just a hint, to help latency a little, any other value will work fine.
+            Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
 
-  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+  At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
   `output->pos` will be updated.
-  Note some content might still be left within internal buffer if `output->size` is too small.
+  Note that some content might still be left within internal buffer if `output->size` is too small.
   @return : nb of bytes still present within internal buffer (0 if it's empty)
             or an error code, which can be tested using ZSTD_isError().
 
@@ -222,20 +214,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
   The epilogue is required for decoders to consider a frame completed.
   Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
   In which case, call again ZSTD_endStream() to complete the flush.
-  @return : nb of bytes still present within internal buffer (0 if it's empty)
+  @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
             or an error code, which can be tested using ZSTD_isError().
 
  
 <BR></pre>
 
-<h3>Streaming compression functions</h3><pre></pre><b><pre>typedef struct ZSTD_CStream_s ZSTD_CStream;
-ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-</pre></b><BR>
 <pre><b>size_t ZSTD_CStreamInSize(void);    </b>/**< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_CStreamOutSize(void);   </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
@@ -261,12 +245,6 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
  
 <BR></pre>
 
-<h3>Streaming decompression functions</h3><pre></pre><b><pre>typedef struct ZSTD_DStream_s ZSTD_DStream;
-ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-size_t ZSTD_initDStream(ZSTD_DStream* zds);
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-</pre></b><BR>
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_DStreamOutSize(void);   </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
@@ -303,10 +281,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 </b></pre><BR>
-<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+<h3>Custom memory allocation functions</h3><pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter12"></a><h2>Advanced compression functions</h2><pre></pre>
 
 <pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
@@ -331,14 +309,14 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </b><p>  Gives the amount of memory used by a given ZSTD_sizeof_CDict 
 </p></pre><BR>
 
-<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
-</b><p>   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+</b><p>   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
-   All fields of `ZSTD_frameParameters` are set to default (0) 
+   `estimatedSrcSize` value is optional, select 0 if not known 
 </p></pre><BR>
 
-<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
-</b><p>   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
+</b><p>   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
-   `srcSize` value is optional, select 0 if not known 
+   All fields of `ZSTD_frameParameters` are set to default (0) 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
@@ -409,22 +387,23 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
 <a name="Chapter14"></a><h2>Advanced streaming functions</h2><pre></pre>
 
-<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+<h3>Advanced Streaming compression functions</h3><pre><b>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct */<b>
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be zero == unknown */<b>
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */<b>
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-</pre></b><BR>
+</b></pre><BR>
-<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+<h3>Advanced Streaming decompression functions</h3><pre><b>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter15"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
@@ -461,13 +440,13 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
   You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 <BR></pre>
 
-<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+<h3>Buffer-less streaming compression functions</h3><pre><b>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter17"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
@@ -511,7 +490,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames == 
+  == Special case : skippable frames ==
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
@@ -530,7 +509,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
     unsigned checksumFlag;
 } ZSTD_frameParams;
 </b></pre><BR>
-<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input, see details below */<b>
+<h3>Buffer-less streaming decompression functions</h3><pre><b>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input, see details below */<b>
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
@@ -538,7 +517,7 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter18"></a><h2>Block functions</h2><pre>
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
@@ -563,10 +542,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() in such a case.
 <BR></pre>
 
-<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+<h3>Raw zstd block functions</h3><pre><b>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
-</pre></b><BR>
+</b></pre><BR>
 </html>
 </body>

lib/common/bitstream.h

@@ -266,7 +266,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
-          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
     } else {
         bitD->start = (const char*)srcBuffer;
@@ -298,7 +298,7 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
 
 MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental */
+#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008  /* experimental */
 #  if defined(__x86_64__)
     if (sizeof(bitContainer)==8)
         return _bextr_u64(bitContainer, start, nbBits);
@@ -367,10 +367,10 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
 }
 
 /*! BIT_reloadDStream() :
-*   Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
+*   Refill `bitD` from buffer previously set in BIT_initDStream() .
 *   This function is safe, it guarantees it will not read beyond src buffer.
 *   @return : status of `BIT_DStream_t` internal register.
-              if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+              if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
 	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should not happen => corruption detected */

lib/common/zstd_common.c

@@ -43,10 +43,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 *   provides error code string from enum */
 const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
 
-/* ---   ZBUFF Error Management  (deprecated)   --- */
-unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
-
 
 /*=**************************************************************
 *  Custom allocator

lib/common/zstd_errors.h

@@ -18,6 +18,20 @@ extern "C" {
 #include <stddef.h>   /* size_t */
 
 
+/* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#  define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+#  define ZSTDERRORLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#endif
+
 /*-****************************************
 *  error codes list
 ******************************************/
@@ -49,8 +63,8 @@ typedef enum {
 /*! ZSTD_getErrorCode() :
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
     which can be used to compare directly with enum list published into "error_public.h" */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
 
 #if defined (__cplusplus)

lib/compress/zstd_compress.c

@@ -2852,6 +2852,8 @@ struct ZSTD_CStream_s {
     ZSTD_cStreamStage stage;
     U32    checksum;
     U32    frameEnded;
+    U64    pledgedSrcSize;
+    U64    inputProcessed;
     ZSTD_parameters params;
     ZSTD_customMem customMem;
 };   /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -2898,7 +2900,7 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
 
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
-    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once */
+    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
 
     if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
@@ -2909,6 +2911,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
     zcs->stage = zcss_load;
     zcs->frameEnded = 0;
+    zcs->pledgedSrcSize = pledgedSrcSize;
+    zcs->inputProcessed = 0;
     return 0;   /* ready to go */
 }
 
@@ -2961,6 +2965,13 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
     return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
 }
 
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
+{
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+    if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
+    return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
+}
+
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 {
     return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
@@ -3057,6 +3068,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
 
     *srcSizePtr = ip - istart;
     *dstCapacityPtr = op - ostart;
+    zcs->inputProcessed += *srcSizePtr;
     if (zcs->frameEnded) return 0;
     {   size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
         if (hintInSize==0) hintInSize = zcs->blockSize;
@@ -3101,6 +3113,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
     BYTE* const oend = (BYTE*)(output->dst) + output->size;
     BYTE* op = ostart;
 
+    if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
+        return ERROR(srcSize_wrong);   /* pledgedSrcSize not respected */
+
     if (zcs->stage != zcss_final) {
         /* flush whatever remains */
         size_t srcSize = 0;

lib/deprecated/zbuff.h

@@ -27,7 +27,7 @@ extern "C" {
 *  Dependencies
 ***************************************/
 #include <stddef.h>      /* size_t */
-#include <zstd.h>        /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
+#include "zstd.h"        /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
 
 
 /* ***************************************************************
@@ -170,7 +170,6 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
 
 
 #ifdef ZBUFF_STATIC_LINKING_ONLY
-
 #ifndef ZBUFF_STATIC_H_30298098432
 #define ZBUFF_STATIC_H_30298098432
 
@@ -203,7 +202,6 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
 
 
 #endif    /* ZBUFF_STATIC_H_30298098432 */
-
 #endif    /* ZBUFF_STATIC_LINKING_ONLY */
 
 

lib/deprecated/zbuff_common.c

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#include "error_private.h"
+#include "zbuff.h"
+
+/*-****************************************
+*  ZBUFF Error Management  (deprecated)
+******************************************/
+
+/*! ZBUFF_isError() :
+*   tells if a return value is an error code */
+unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
+/*! ZBUFF_getErrorName() :
+*   provides error code string from function result (useful for debugging) */
+const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+

lib/dictBuilder/zdict.c

@@ -36,12 +36,11 @@
 #include <time.h>          /* clock */
 
 #include "mem.h"           /* read */
-#include "error_private.h"
 #include "fse.h"           /* FSE_normalizeCount, FSE_writeNCount */
 #define HUF_STATIC_LINKING_ONLY
-#include "huf.h"
+#include "huf.h"           /* HUF_buildCTable, HUF_writeCTable */
 #include "zstd_internal.h" /* includes zstd.h */
-#include "xxhash.h"
+#include "xxhash.h"        /* XXH64 */
 #include "divsufsort.h"
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #  define ZDICT_STATIC_LINKING_ONLY
@@ -570,7 +569,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
             if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
     }
     cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
-    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
     if (cSize) {  /* if == 0; block is not compressible */
         const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);

lib/dictBuilder/zdict.h

@@ -19,15 +19,18 @@ extern "C" {
 #include <stddef.h>  /* size_t */
 
 
-/*======  Export for Windows  ======*/
+/* =====   ZDICTLIB_API : control library symbols visibility   ===== */
-/*!
+#if defined(__GNUC__) && (__GNUC__ >= 4)
-*  ZSTD_DLL_EXPORT :
+#  define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
-*  Enable exporting of functions when building a Windows DLL
-*/
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZDICTLIB_API __declspec(dllexport)
 #else
-#  define ZDICTLIB_API
+#  define ZDICTLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZDICTLIB_API ZDICTLIB_VISIBILITY
 #endif
 
 
@@ -79,7 +82,7 @@ typedef struct {
               or an error code, which can be tested by ZDICT_isError().
     note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
 */
-size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
                                 const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                                 ZDICT_params_t parameters);
 
@@ -97,7 +100,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
     starting from its beginning.
     @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
 */
-size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                         const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 

lib/zstd.h

@@ -20,13 +20,16 @@ extern "C" {
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
 #if defined(__GNUC__) && (__GNUC__ >= 4)
-#  define ZSTDLIB_API __attribute__ ((visibility ("default")))
+#  define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
-#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZSTDLIB_API __declspec(dllexport)
-#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define ZSTDLIB_API
+#  define ZSTDLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDLIB_API ZSTDLIB_VISIBILITY
 #endif
 
 
@@ -53,7 +56,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    1
-#define ZSTD_VERSION_RELEASE  2
+#define ZSTD_VERSION_RELEASE  3
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -237,20 +240,20 @@ typedef struct ZSTD_outBuffer_s {
 *
 *  Start a new compression by initializing ZSTD_CStream.
 *  Use ZSTD_initCStream() to start a new compression operation.
-*  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+*  Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
 *
 *  Use ZSTD_compressStream() repetitively to consume input stream.
 *  The function will automatically update both `pos` fields.
 *  Note that it may not consume the entire input, in which case `pos < size`,
 *  and it's up to the caller to present again remaining data.
 *  @return : a size hint, preferred nb of bytes to use as input for next function call
-*           (it's just a hint, to help latency a little, any other value will work fine)
-*           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
 *            or an error code, which can be tested using ZSTD_isError().
+*            Note 1 : it's just a hint, to help latency a little, any other value will work fine.
+*            Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
 *
-*  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+*  At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
 *  `output->pos` will be updated.
-*  Note some content might still be left within internal buffer if `output->size` is too small.
+*  Note that some content might still be left within internal buffer if `output->size` is too small.
 *  @return : nb of bytes still present within internal buffer (0 if it's empty)
 *            or an error code, which can be tested using ZSTD_isError().
 *
@@ -259,15 +262,15 @@ typedef struct ZSTD_outBuffer_s {
 *  The epilogue is required for decoders to consider a frame completed.
 *  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
 *  In which case, call again ZSTD_endStream() to complete the flush.
-*  @return : nb of bytes still present within internal buffer (0 if it's empty)
+*  @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
 *            or an error code, which can be tested using ZSTD_isError().
 *
 * *******************************************************************/
 
-/*=====   Streaming compression functions   ======*/
 typedef struct ZSTD_CStream_s ZSTD_CStream;
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -300,10 +303,10 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 *            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 * *******************************************************************************/
 
-/*=====   Streaming decompression functions   =====*/
 typedef struct ZSTD_DStream_s ZSTD_DStream;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
@@ -406,15 +409,15 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
  *  Gives the amount of memory used by a given ZSTD_sizeof_CDict */
 ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
 
-/*! ZSTD_getParams() :
-*   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
-*   All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
-
 /*! ZSTD_getCParams() :
-*   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
+*   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
-*   `srcSize` value is optional, select 0 if not known */
+*   `estimatedSrcSize` value is optional, select 0 if not known */
-ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_getParams() :
+*   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+*   All fields of `ZSTD_frameParameters` are set to default (0) */
+ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 
 /*! ZSTD_checkCParams() :
 *   Ensure param values remain within authorized range */
@@ -463,13 +466,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  *  Provides the dictID stored within dictionary.
  *  if @return == 0, the dictionary is not conformant with Zstandard specification.
  *  It can still be loaded, but as a content-only dictionary. */
-unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
 
 /*! ZSTD_getDictID_fromDDict() :
  *  Provides the dictID of the dictionary loaded into `ddict`.
  *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
  *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
 
 /*! ZSTD_getDictID_fromFrame() :
  *  Provides the dictID required to decompressed the frame stored within `src`.
@@ -481,7 +484,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
  *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
  *  - This is not a Zstandard frame.
  *  When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
-unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 
 /********************************************************************
@@ -490,6 +493,7 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
@@ -602,7 +606,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames == 
+  == Special case : skippable frames ==
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :

programs/dibio.c

@@ -31,7 +31,8 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#define MEMMULT 11
+#define SAMPLESIZE_MAX (128 KB)
+#define MEMMULT 11    /* rough estimation : memory cost to analyze 1 byte of sample */
 static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
 
 #define NOISELENGTH 32
@@ -97,7 +98,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
     for (n=0; n<nbFiles; n++) {
         const char* const fileName = fileNamesTable[n];
         unsigned long long const fs64 = UTIL_getFileSize(fileName);
-        size_t const fileSize = (size_t) MIN(fs64, 128 KB);
+        size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX);
         if (fileSize > *bufferSizePtr-pos) break;
         {   FILE* const f = fopen(fileName, "rb");
             if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
@@ -163,6 +164,21 @@ static void DiB_saveDict(const char* dictFileName,
 }
 
 
+static int g_tooLargeSamples = 0;
+static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+    U64 total = 0;
+    unsigned n;
+    for (n=0; n<nbFiles; n++) {
+        U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        U64 const cappedFileSize = MIN(fileSize, SAMPLESIZE_MAX);
+        total += cappedFileSize;
+        g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX);
+    }
+    return total;
+}
+
+
 /*! ZDICT_trainFromBuffer_unsafe() :
     Strictly Internal use only !!
     Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
@@ -181,16 +197,21 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
 {
     void* const dictBuffer = malloc(maxDictSize);
     size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
-    unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
+    unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
     size_t const maxMem =  DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
-    size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
+    size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
     void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
     int result = 0;
 
     /* Checks */
     if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles");   /* should not happen */
     g_displayLevel = params.notificationLevel;
-    if (nbFiles < 5) {
+    if (g_tooLargeSamples) {
+        DISPLAYLEVEL(2, "!  Warning : some samples are very large \n");
+        DISPLAYLEVEL(2, "!  Note that dictionary is only useful for small files or beginning of large files. \n");
+        DISPLAYLEVEL(2, "!  As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX);
+    }
+    if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) {
         DISPLAYLEVEL(2, "!  Warning : nb of samples too low for proper processing ! \n");
         DISPLAYLEVEL(2, "!  Please provide _one file per sample_. \n");
         DISPLAYLEVEL(2, "!  Do not concatenate samples together into a single file, \n");

tests/.gitignore

@@ -3,15 +3,18 @@ fullbench
 fullbench32
 fuzzer
 fuzzer32
+fuzzer-dll
 zbufftest
 zbufftest32
 zstreamtest
 zstreamtest32
+zstreamtest-dll
 datagen
 paramgrill
 paramgrill32
 roundTripCrash
 longmatch
+symbols
 
 # Tmp test directory
 zstdtest

tests/Makefile

@@ -28,12 +28,12 @@ PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
-CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR)
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
-CFLAGS ?= -O3
+CFLAGS  ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
-CFLAGS += $(MOREFLAGS)
+CFLAGS  += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
@@ -57,7 +57,7 @@ ZSTREAM_TESTTIME = -T2mn
 FUZZERTEST= -T5mn
 ZSTDRTTEST= --test-large-data
 
-.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest
+.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
 
 default: fullbench
 
@@ -65,6 +65,8 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
 
 all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
 
+dll: fuzzer-dll zstreamtest-dll zbufftest-dll
+
 
 
 zstd:
@@ -93,28 +95,45 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 
-fuzzer  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
+fuzzer   : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
+fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
 zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
-zbufftest : CFLAGS += -Wno-deprecated-declarations
+zbufftest : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
 zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)   # flag required to silence deprecation warnings
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
 zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
+zbufftest-dll : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
+zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
+
 zstreamtest  : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
 	$(CC) -m32  $(FLAGS) $^ -o $@$(EXT)
 
+zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
+
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
 
@@ -127,6 +146,15 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
 longmatch  : $(ZSTD_FILES) longmatch.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
+symbols  : symbols.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+ifneq (,$(filter Windows%,$(OS)))
+	cp $(ZSTDDIR)/dll/libzstd.dll .
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll
+else
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
+endif
+
 namespaceTest:
 	if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
 	$(RM) $@
@@ -142,8 +170,10 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
-		zstreamtest$(EXT) zstreamtest32$(EXT) \
+        fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
-        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
+        zstreamtest$(EXT) zstreamtest32$(EXT) \
+        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
+        symbols$(EXT)
 	@echo Cleaning completed
 
 
@@ -243,4 +273,7 @@ test-zstream32: zstreamtest32
 test-longmatch: longmatch
 	$(QEMU_SYS) ./longmatch
 
+test-symbols: symbols
+	$(QEMU_SYS) ./symbols
+
 endif

tests/symbols.c

@@ -0,0 +1,144 @@
+#include <stdio.h>
+#include "zstd_errors.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#define ZBUFF_DISABLE_DEPRECATE_WARNINGS
+#define ZBUFF_STATIC_LINKING_ONLY
+#include "zbuff.h"
+#define ZDICT_STATIC_LINKING_ONLY
+#include "zdict.h"
+
+static const void *symbols[] = {
+/* zstd.h */
+  &ZSTD_versionNumber,
+  &ZSTD_compress,
+  &ZSTD_decompress,
+  &ZSTD_getDecompressedSize,
+  &ZSTD_maxCLevel,
+  &ZSTD_compressBound,
+  &ZSTD_isError,
+  &ZSTD_getErrorName,
+  &ZSTD_createCCtx,
+  &ZSTD_freeCCtx,
+  &ZSTD_compressCCtx,
+  &ZSTD_createDCtx,
+  &ZSTD_freeDCtx,
+  &ZSTD_decompressDCtx,
+  &ZSTD_compress_usingDict,
+  &ZSTD_decompress_usingDict,
+  &ZSTD_createCDict,
+  &ZSTD_freeCDict,
+  &ZSTD_compress_usingCDict,
+  &ZSTD_createDDict,
+  &ZSTD_freeDDict,
+  &ZSTD_decompress_usingDDict,
+  &ZSTD_createCStream,
+  &ZSTD_freeCStream,
+  &ZSTD_initCStream,
+  &ZSTD_compressStream,
+  &ZSTD_flushStream,
+  &ZSTD_endStream,
+  &ZSTD_CStreamInSize,
+  &ZSTD_CStreamOutSize,
+  &ZSTD_createDStream,
+  &ZSTD_freeDStream,
+  &ZSTD_initDStream,
+  &ZSTD_decompressStream,
+  &ZSTD_DStreamInSize,
+  &ZSTD_DStreamOutSize,
+/* zstd.h: advanced functions */
+  &ZSTD_estimateCCtxSize,
+  &ZSTD_createCCtx_advanced,
+  &ZSTD_sizeof_CCtx,
+  &ZSTD_createCDict_advanced,
+  &ZSTD_sizeof_CDict,
+  &ZSTD_getCParams,
+  &ZSTD_getParams,
+  &ZSTD_checkCParams,
+  &ZSTD_adjustCParams,
+  &ZSTD_compress_advanced,
+  &ZSTD_isFrame,
+  &ZSTD_estimateDCtxSize,
+  &ZSTD_createDCtx_advanced,
+  &ZSTD_sizeof_DCtx,
+  &ZSTD_sizeof_DDict,
+  &ZSTD_getDictID_fromDict,
+  &ZSTD_getDictID_fromDDict,
+  &ZSTD_getDictID_fromFrame,
+  &ZSTD_createCStream_advanced,
+  &ZSTD_initCStream_srcSize,
+  &ZSTD_initCStream_usingDict,
+  &ZSTD_initCStream_advanced,
+  &ZSTD_initCStream_usingCDict,
+  &ZSTD_resetCStream,
+  &ZSTD_sizeof_CStream,
+  &ZSTD_createDStream_advanced,
+  &ZSTD_initDStream_usingDict,
+  &ZSTD_setDStreamParameter,
+  &ZSTD_initDStream_usingDDict,
+  &ZSTD_resetDStream,
+  &ZSTD_sizeof_DStream,
+  &ZSTD_compressBegin,
+  &ZSTD_compressBegin_usingDict,
+  &ZSTD_compressBegin_advanced,
+  &ZSTD_copyCCtx,
+  &ZSTD_compressContinue,
+  &ZSTD_compressEnd,
+  &ZSTD_getFrameParams,
+  &ZSTD_decompressBegin,
+  &ZSTD_decompressBegin_usingDict,
+  &ZSTD_copyDCtx,
+  &ZSTD_nextSrcSizeToDecompress,
+  &ZSTD_decompressContinue,
+  &ZSTD_nextInputType,
+  &ZSTD_getBlockSizeMax,
+  &ZSTD_compressBlock,
+  &ZSTD_decompressBlock,
+  &ZSTD_insertBlock,
+/* zstd_errors.h */
+  &ZSTD_getErrorCode,
+  &ZSTD_getErrorString,
+/* zbuff.h */
+  &ZBUFF_createCCtx,
+  &ZBUFF_freeCCtx,
+  &ZBUFF_compressInit,
+  &ZBUFF_compressInitDictionary,
+  &ZBUFF_compressContinue,
+  &ZBUFF_compressFlush,
+  &ZBUFF_compressEnd,
+  &ZBUFF_createDCtx,
+  &ZBUFF_freeDCtx,
+  &ZBUFF_decompressInit,
+  &ZBUFF_decompressInitDictionary,
+  &ZBUFF_decompressContinue,
+  &ZBUFF_isError,
+  &ZBUFF_getErrorName,
+  &ZBUFF_recommendedCInSize,
+  &ZBUFF_recommendedCOutSize,
+  &ZBUFF_recommendedDInSize,
+  &ZBUFF_recommendedDOutSize,
+/* zbuff.h: advanced functions */
+  &ZBUFF_createCCtx_advanced,
+  &ZBUFF_createDCtx_advanced,
+  &ZBUFF_compressInit_advanced,
+/* zdict.h */
+  &ZDICT_trainFromBuffer,
+  &ZDICT_getDictID,
+  &ZDICT_isError,
+  &ZDICT_getErrorName,
+/* zdict.h: advanced functions */
+  &ZDICT_trainFromBuffer_advanced,
+  &ZDICT_addEntropyTablesFromBuffer,
+  NULL,
+};
+
+int main(int argc, const char** argv) {
+  const void **symbol;
+  (void)argc;
+  (void)argv;
+
+  for (symbol = symbols; *symbol != NULL; ++symbol) {
+    printf("%p\n", *symbol);
+  }
+  return 0;
+}

tests/zbufftest.c

@@ -28,8 +28,8 @@
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
 #include "zstd.h"         /* ZSTD_compressBound */
-#define ZBUFF_STATIC_LINKING_ONLY
+#define ZBUFF_STATIC_LINKING_ONLY  /* ZBUFF_createCCtx_advanced */
-#include "zbuff.h"        /* ZBUFF_createCCtx_advanced */
+#include "zbuff.h"        /* ZBUFF_isError */
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"       /* XXH64_* */
@@ -265,11 +265,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     static const U32 maxSrcLog = 24;
     static const U32 maxSampleLog = 19;
     BYTE* cNoiseBuffer[5];
-    size_t srcBufferSize = (size_t)1<<maxSrcLog;
+    size_t const srcBufferSize = (size_t)1<<maxSrcLog;
     BYTE* copyBuffer;
-    size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+    size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
     BYTE* cBuffer;
-    size_t cBufferSize   = ZSTD_compressBound(srcBufferSize);
+    size_t const cBufferSize   = ZSTD_compressBound(srcBufferSize);
     BYTE* dstBuffer;
     size_t dstBufferSize = srcBufferSize;
     U32 result = 0;

tests/zstreamtest.c

@@ -28,6 +28,7 @@
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel, ZSTD_customMem */
 #include "zstd.h"         /* ZSTD_compressBound */
+#include "zstd_errors.h"  /* ZSTD_error_srcSize_wrong */
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64_* */
@@ -88,16 +89,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
     return rand32 >> 5;
 }
 
-/*
-static unsigned FUZ_highbit32(U32 v32)
-{
-    unsigned nbBits = 0;
-    if (v32==0) return 0;
-    for ( ; v32 ; v32>>=1) nbBits++;
-    return nbBits;
-}
-*/
-
 static void* allocFunction(void* opaque, size_t size)
 {
     void* address = malloc(size);
@@ -254,6 +245,42 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }   }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* _srcSize compression test */
+    DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
+    outBuff.dst = (char*)(compressedBuffer);
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+    { unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
+      DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos);
+      DISPLAY("origSize = %u \n", (U32)origSize);
+      if ((size_t)origSize != CNBufferSize) goto _output_error; }  /* exact original size must be present */
+    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+
+    /* wrong _srcSize compression test */
+    DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
+    ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
+    outBuff.dst = (char*)(compressedBuffer);
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
+      DISPLAYLEVEL(4, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
+
     /* Complex context re-use scenario */
     DISPLAYLEVEL(4, "test%3i : context re-use : ", testNb++);
     ZSTD_freeCStream(zc);
@@ -519,7 +546,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
             U32 n;
             for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
-                /* compress random chunk into random size dst buffer */
+                /* compress random chunks into randomly sized dst buffers */
                 {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
                     size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
                     size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);

zlibWrapper/examples/fitblk.c

@@ -90,7 +90,7 @@ local int partcompress(FILE *in, z_streamp def)
             flush = Z_FINISH;
         LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         ret = deflate(def, flush);
-        LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+        LOG_FITBLK("partcompress2 ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         assert(ret != Z_STREAM_ERROR);
     } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
     return ret;
@@ -106,6 +106,7 @@ local int recompress(z_streamp inf, z_streamp def)
     unsigned char raw[RAWLEN];
 
     flush = Z_NO_FLUSH;
+    LOG_FITBLK("recompress start\n");
     do {
         /* decompress */
         inf->avail_out = RAWLEN;
@@ -125,7 +126,7 @@ local int recompress(z_streamp inf, z_streamp def)
             flush = Z_FINISH;
         LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         ret = deflate(def, flush);
-        LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+        LOG_FITBLK("recompress2deflate ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         assert(ret != Z_STREAM_ERROR);
     } while (ret != Z_STREAM_END && def->avail_out != 0);
     return ret;
@@ -165,9 +166,6 @@ int main(int argc, char **argv)
     ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
     if (ret != Z_OK || blk == NULL)
         quit("out of memory");
-    ret = ZWRAP_setPledgedSrcSize(&def, 1<<16);
-    if (ret != Z_OK)
-        quit("ZWRAP_setPledgedSrcSize");
 
     /* compress from stdin until output full, or no more input */
     def.avail_out = size + EXCESS;

zlibWrapper/examples/minigzip.c

@@ -3,7 +3,7 @@
 
 /* minigzip.c -- simulate gzip using the zlib compression library
  * Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly.
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 /*

zlibWrapper/gzclose.c

@@ -3,7 +3,7 @@
 
 /* gzclose.c -- zlib gzclose() function
  * Copyright (C) 2004, 2010 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"

zlibWrapper/gzguts.h

@@ -4,7 +4,7 @@
 
 /* gzguts.h -- zlib internal header definitions for gz* operations
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #ifdef _LARGEFILE64_SOURCE

zlibWrapper/gzlib.c

@@ -3,7 +3,7 @@
 
 /* gzlib.c -- zlib functions common to reading and writing gzip files
  * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"

zlibWrapper/gzread.c

@@ -3,7 +3,7 @@
 
 /* gzread.c -- zlib functions for reading gzip files
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"

zlibWrapper/gzwrite.c

@@ -3,7 +3,7 @@
 
 /* gzwrite.c -- zlib functions for writing gzip files
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"

zlibWrapper/zstd_zlibwrapper.c

@@ -301,7 +301,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         }
     }
 
-    LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+    LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
     if (strm->avail_in > 0) {
         zwc->inBuffer.src = strm->next_in;
         zwc->inBuffer.size = strm->avail_in;
@@ -355,7 +355,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         strm->total_out += zwc->outBuffer.pos;
         strm->avail_out -= zwc->outBuffer.pos;
     }
-    LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+    LOG_WRAPPERC("- deflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
     return Z_OK;
 }