Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
This commit is contained in:
Przemyslaw Skibinski
commit
5477dd7ac9
24
.travis.yml
24
.travis.yml
@ -12,7 +12,7 @@ matrix:
|
||||
os: linux
|
||||
sudo: false
|
||||
|
||||
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
|
||||
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
|
||||
os: linux
|
||||
sudo: false
|
||||
language: cpp
|
||||
@ -69,7 +69,7 @@ matrix:
|
||||
|
||||
|
||||
# Ubuntu 14.04 LTS Server Edition 64 bit
|
||||
- env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
|
||||
- env: Ubu=14.04 Cmd="make armtest"
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
@ -78,7 +78,17 @@ matrix:
|
||||
- qemu-system-arm
|
||||
- qemu-user-static
|
||||
- gcc-arm-linux-gnueabi
|
||||
- libc6-dev-armel-cross
|
||||
- libc6-dev-armel-cross
|
||||
|
||||
# Ubuntu 14.04 LTS Server Edition 64 bit
|
||||
- env: Ubu=14.04 Cmd="make aarch64test"
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- qemu-system-arm
|
||||
- qemu-user-static
|
||||
- gcc-aarch64-linux-gnu
|
||||
- libc6-dev-arm64-cross
|
||||
|
||||
@ -91,6 +101,7 @@ matrix:
|
||||
- qemu-system-ppc
|
||||
- qemu-user-static
|
||||
- gcc-powerpc-linux-gnu
|
||||
- libc6-dev-armel-cross
|
||||
|
||||
- env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
|
||||
os: linux
|
||||
@ -101,7 +112,10 @@ matrix:
|
||||
packages:
|
||||
- valgrind
|
||||
|
||||
- env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
|
||||
- env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean
|
||||
&& make c99test && make clean && make gnu99test && make clean
|
||||
&& make clangtest && make clean && make -C contrib/pzstd googletest32
|
||||
&& make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
@ -127,7 +141,7 @@ matrix:
|
||||
- libc6-dev-i386
|
||||
- gcc-multilib
|
||||
|
||||
- env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
|
||||
- env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll"
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
|
||||
8
NEWS
8
NEWS
@ -1,6 +1,10 @@
|
||||
v1.1.3
|
||||
cli : new : commands for advanced parameters, by Przemyslaw Skibinski
|
||||
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
|
||||
|
||||
v1.1.2
|
||||
API : streaming : decompression : changed : implicit reset on starting new frames without init
|
||||
API : experimental : added : dictID retrieval functions
|
||||
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
|
||||
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
|
||||
API : zbuff : changed : prototypes now generate deprecation warnings
|
||||
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
|
||||
lib : changed : only public ZSTD_ symbols are now exposed
|
||||
|
||||
@ -63,7 +63,8 @@ build_script:
|
||||
make -v &&
|
||||
cc -v &&
|
||||
ECHO make %MAKE_PARAMS% &&
|
||||
make %MAKE_PARAMS%
|
||||
make %MAKE_PARAMS% &&
|
||||
make -C tests test-symbols
|
||||
)
|
||||
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
|
||||
COPY programs\zstd.exe bin\zstd.exe &&
|
||||
|
||||
@ -359,6 +359,10 @@
|
||||
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
|
||||
>
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
|
||||
@ -69,6 +69,7 @@ SET(Sources
|
||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_compress.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
|
||||
|
||||
|
||||
@ -88,29 +88,21 @@
|
||||
note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Helper functions</h3><pre></pre><b><pre>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
|
||||
<h3>Helper functions</h3><pre><b>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
|
||||
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
|
||||
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
|
||||
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
<a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
|
||||
|
||||
<h3>Compression context</h3><pre> When compressing many times,
|
||||
it is recommended to allocate a context just once, and re-use it for each successive compression operation.
|
||||
This will make workload friendlier for system's memory.
|
||||
Use one context per thread for parallel execution in multi-threaded environments.
|
||||
</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
|
||||
ZSTD_CCtx* ZSTD_createCCtx(void);
|
||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
|
||||
</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Decompression context</h3><pre></pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||
<h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||
ZSTD_DCtx* ZSTD_createDCtx(void);
|
||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
<pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()).
|
||||
</p></pre><BR>
|
||||
@ -200,20 +192,20 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
|
||||
Start a new compression by initializing ZSTD_CStream.
|
||||
Use ZSTD_initCStream() to start a new compression operation.
|
||||
Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
|
||||
Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
|
||||
|
||||
Use ZSTD_compressStream() repetitively to consume input stream.
|
||||
The function will automatically update both `pos` fields.
|
||||
Note that it may not consume the entire input, in which case `pos < size`,
|
||||
and it's up to the caller to present again remaining data.
|
||||
@return : a size hint, preferred nb of bytes to use as input for next function call
|
||||
(it's just a hint, to help latency a little, any other value will work fine)
|
||||
(note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
|
||||
or an error code, which can be tested using ZSTD_isError().
|
||||
Note 1 : it's just a hint, to help latency a little, any other value will work fine.
|
||||
Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
|
||||
|
||||
At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
|
||||
At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
|
||||
`output->pos` will be updated.
|
||||
Note some content might still be left within internal buffer if `output->size` is too small.
|
||||
Note that some content might still be left within internal buffer if `output->size` is too small.
|
||||
@return : nb of bytes still present within internal buffer (0 if it's empty)
|
||||
or an error code, which can be tested using ZSTD_isError().
|
||||
|
||||
@ -222,20 +214,12 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
The epilogue is required for decoders to consider a frame completed.
|
||||
Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
|
||||
In which case, call again ZSTD_endStream() to complete the flush.
|
||||
@return : nb of bytes still present within internal buffer (0 if it's empty)
|
||||
@return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
|
||||
or an error code, which can be tested using ZSTD_isError().
|
||||
|
||||
|
||||
<BR></pre>
|
||||
|
||||
<h3>Streaming compression functions</h3><pre></pre><b><pre>typedef struct ZSTD_CStream_s ZSTD_CStream;
|
||||
ZSTD_CStream* ZSTD_createCStream(void);
|
||||
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
||||
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
||||
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
||||
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b>
|
||||
</b></pre><BR>
|
||||
<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
|
||||
@ -261,12 +245,6 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
||||
|
||||
<BR></pre>
|
||||
|
||||
<h3>Streaming decompression functions</h3><pre></pre><b><pre>typedef struct ZSTD_DStream_s ZSTD_DStream;
|
||||
ZSTD_DStream* ZSTD_createDStream(void);
|
||||
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||
size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
||||
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_DStreamInSize(void); </b>/*!< recommended size for input buffer */<b>
|
||||
</b></pre><BR>
|
||||
<pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
|
||||
@ -303,10 +281,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
ZSTD_frameParameters fParams;
|
||||
} ZSTD_parameters;
|
||||
</b></pre><BR>
|
||||
<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
<h3>Custom memory allocation functions</h3><pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
<a name="Chapter12"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
|
||||
@ -331,14 +309,14 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
</b><p> Gives the amount of memory used by a given ZSTD_sizeof_CDict
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
|
||||
</b><p> same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
|
||||
All fields of `ZSTD_frameParameters` are set to default (0)
|
||||
<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
||||
</b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
|
||||
`estimatedSrcSize` value is optional, select 0 if not known
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
|
||||
</b><p> @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||
`srcSize` value is optional, select 0 if not known
|
||||
<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
||||
</b><p> same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
|
||||
All fields of `ZSTD_frameParameters` are set to default (0)
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||
@ -409,22 +387,23 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
|
||||
<a name="Chapter14"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
|
||||
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
<h3>Advanced Streaming compression functions</h3><pre><b>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct */<b>
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be zero == unknown */<b>
|
||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */<b>
|
||||
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
||||
</pre></b><BR>
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
</b></pre><BR>
|
||||
<h3>Advanced Streaming decompression functions</h3><pre><b>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
||||
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
|
||||
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
<a name="Chapter15"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
||||
But it's also a complex one, with many restrictions (documented below).
|
||||
@ -461,13 +440,13 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
<h3>Buffer-less streaming compression functions</h3><pre><b>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
|
||||
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
<a name="Chapter17"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
@ -511,7 +490,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
|
||||
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
||||
This information is not required to properly decode a frame.
|
||||
|
||||
== Special case : skippable frames ==
|
||||
== Special case : skippable frames ==
|
||||
|
||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
|
||||
@ -530,7 +509,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
|
||||
unsigned checksumFlag;
|
||||
} ZSTD_frameParams;
|
||||
</b></pre><BR>
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre><b>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
@ -538,7 +517,7 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
<a name="Chapter18"></a><h2>Block functions</h2><pre>
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
@ -563,10 +542,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
Use ZSTD_insertBlock() in such a case.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
|
||||
<h3>Raw zstd block functions</h3><pre><b>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
|
||||
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
|
||||
</pre></b><BR>
|
||||
</b></pre><BR>
|
||||
</html>
|
||||
</body>
|
||||
|
||||
@ -266,7 +266,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
} else {
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
@ -298,7 +298,7 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */
|
||||
# if defined(__x86_64__)
|
||||
if (sizeof(bitContainer)==8)
|
||||
return _bextr_u64(bitContainer, start, nbBits);
|
||||
@ -367,10 +367,10 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStream() :
|
||||
* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
|
||||
* Refill `bitD` from buffer previously set in BIT_initDStream() .
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BIT_DStream_t` internal register.
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
|
||||
|
||||
@ -43,10 +43,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
|
||||
/* --- ZBUFF Error Management (deprecated) --- */
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
|
||||
/*=**************************************************************
|
||||
* Custom allocator
|
||||
|
||||
@ -18,6 +18,20 @@ extern "C" {
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
|
||||
#if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
#else
|
||||
# define ZSTDERRORLIB_VISIBILITY
|
||||
#endif
|
||||
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
/*-****************************************
|
||||
* error codes list
|
||||
******************************************/
|
||||
@ -49,8 +63,8 @@ typedef enum {
|
||||
/*! ZSTD_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
||||
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
@ -2852,6 +2852,8 @@ struct ZSTD_CStream_s {
|
||||
ZSTD_cStreamStage stage;
|
||||
U32 checksum;
|
||||
U32 frameEnded;
|
||||
U64 pledgedSrcSize;
|
||||
U64 inputProcessed;
|
||||
ZSTD_parameters params;
|
||||
ZSTD_customMem customMem;
|
||||
}; /* typedef'd to ZSTD_CStream within "zstd.h" */
|
||||
@ -2898,7 +2900,7 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
|
||||
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */
|
||||
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
|
||||
|
||||
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
|
||||
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
|
||||
@ -2909,6 +2911,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
|
||||
zcs->stage = zcss_load;
|
||||
zcs->frameEnded = 0;
|
||||
zcs->pledgedSrcSize = pledgedSrcSize;
|
||||
zcs->inputProcessed = 0;
|
||||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
@ -2961,6 +2965,13 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
|
||||
return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
|
||||
if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
|
||||
return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
|
||||
{
|
||||
return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
|
||||
@ -3057,6 +3068,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
|
||||
*srcSizePtr = ip - istart;
|
||||
*dstCapacityPtr = op - ostart;
|
||||
zcs->inputProcessed += *srcSizePtr;
|
||||
if (zcs->frameEnded) return 0;
|
||||
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
|
||||
if (hintInSize==0) hintInSize = zcs->blockSize;
|
||||
@ -3101,6 +3113,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
|
||||
BYTE* const oend = (BYTE*)(output->dst) + output->size;
|
||||
BYTE* op = ostart;
|
||||
|
||||
if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
|
||||
return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
|
||||
|
||||
if (zcs->stage != zcss_final) {
|
||||
/* flush whatever remains */
|
||||
size_t srcSize = 0;
|
||||
|
||||
@ -27,7 +27,7 @@ extern "C" {
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <zstd.h> /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
||||
#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
@ -170,7 +170,6 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
|
||||
|
||||
|
||||
#ifdef ZBUFF_STATIC_LINKING_ONLY
|
||||
|
||||
#ifndef ZBUFF_STATIC_H_30298098432
|
||||
#define ZBUFF_STATIC_H_30298098432
|
||||
|
||||
@ -203,7 +202,6 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
|
||||
|
||||
|
||||
#endif /* ZBUFF_STATIC_H_30298098432 */
|
||||
|
||||
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
|
||||
26
lib/deprecated/zbuff_common.c
Normal file
26
lib/deprecated/zbuff_common.c
Normal file
@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "error_private.h"
|
||||
#include "zbuff.h"
|
||||
|
||||
/*-****************************************
|
||||
* ZBUFF Error Management (deprecated)
|
||||
******************************************/
|
||||
|
||||
/*! ZBUFF_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
/*! ZBUFF_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
@ -36,12 +36,11 @@
|
||||
#include <time.h> /* clock */
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "error_private.h"
|
||||
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "xxhash.h"
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
#include "divsufsort.h"
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
@ -570,7 +569,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
||||
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
|
||||
}
|
||||
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
|
||||
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
|
||||
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
|
||||
|
||||
if (cSize) { /* if == 0; block is not compressible */
|
||||
const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
|
||||
|
||||
@ -19,15 +19,18 @@ extern "C" {
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/*====== Export for Windows ======*/
|
||||
/*!
|
||||
* ZSTD_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL
|
||||
*/
|
||||
#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZDICTLIB_API __declspec(dllexport)
|
||||
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
|
||||
#if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
#else
|
||||
# define ZDICTLIB_API
|
||||
# define ZDICTLIB_VISIBILITY
|
||||
#endif
|
||||
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
|
||||
@ -79,7 +82,7 @@ typedef struct {
|
||||
or an error code, which can be tested by ZDICT_isError().
|
||||
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
@ -97,7 +100,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
|
||||
starting from its beginning.
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
|
||||
*/
|
||||
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||
ZDICTLIB_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
||||
|
||||
|
||||
|
||||
58
lib/zstd.h
58
lib/zstd.h
@ -20,13 +20,16 @@ extern "C" {
|
||||
|
||||
/* ===== ZSTDLIB_API : control library symbols visibility ===== */
|
||||
#if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define ZSTDLIB_API __attribute__ ((visibility ("default")))
|
||||
#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport)
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
#else
|
||||
# define ZSTDLIB_API
|
||||
# define ZSTDLIB_VISIBILITY
|
||||
#endif
|
||||
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define ZSTDLIB_API ZSTDLIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
|
||||
@ -53,7 +56,7 @@ extern "C" {
|
||||
/*------ Version ------*/
|
||||
#define ZSTD_VERSION_MAJOR 1
|
||||
#define ZSTD_VERSION_MINOR 1
|
||||
#define ZSTD_VERSION_RELEASE 2
|
||||
#define ZSTD_VERSION_RELEASE 3
|
||||
|
||||
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
||||
#define ZSTD_QUOTE(str) #str
|
||||
@ -237,20 +240,20 @@ typedef struct ZSTD_outBuffer_s {
|
||||
*
|
||||
* Start a new compression by initializing ZSTD_CStream.
|
||||
* Use ZSTD_initCStream() to start a new compression operation.
|
||||
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
|
||||
* Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
|
||||
*
|
||||
* Use ZSTD_compressStream() repetitively to consume input stream.
|
||||
* The function will automatically update both `pos` fields.
|
||||
* Note that it may not consume the entire input, in which case `pos < size`,
|
||||
* and it's up to the caller to present again remaining data.
|
||||
* @return : a size hint, preferred nb of bytes to use as input for next function call
|
||||
* (it's just a hint, to help latency a little, any other value will work fine)
|
||||
* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
|
||||
* or an error code, which can be tested using ZSTD_isError().
|
||||
* Note 1 : it's just a hint, to help latency a little, any other value will work fine.
|
||||
* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
|
||||
*
|
||||
* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
|
||||
* At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
|
||||
* `output->pos` will be updated.
|
||||
* Note some content might still be left within internal buffer if `output->size` is too small.
|
||||
* Note that some content might still be left within internal buffer if `output->size` is too small.
|
||||
* @return : nb of bytes still present within internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZSTD_isError().
|
||||
*
|
||||
@ -259,15 +262,15 @@ typedef struct ZSTD_outBuffer_s {
|
||||
* The epilogue is required for decoders to consider a frame completed.
|
||||
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
|
||||
* In which case, call again ZSTD_endStream() to complete the flush.
|
||||
* @return : nb of bytes still present within internal buffer (0 if it's empty)
|
||||
* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
|
||||
* or an error code, which can be tested using ZSTD_isError().
|
||||
*
|
||||
* *******************************************************************/
|
||||
|
||||
/*===== Streaming compression functions ======*/
|
||||
typedef struct ZSTD_CStream_s ZSTD_CStream;
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
||||
@ -300,10 +303,10 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
||||
* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
|
||||
* *******************************************************************************/
|
||||
|
||||
/*===== Streaming decompression functions =====*/
|
||||
typedef struct ZSTD_DStream_s ZSTD_DStream;
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
||||
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
|
||||
@ -406,15 +409,15 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
|
||||
* Gives the amount of memory used by a given ZSTD_sizeof_CDict */
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
||||
|
||||
/*! ZSTD_getParams() :
|
||||
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
|
||||
* All fields of `ZSTD_frameParameters` are set to default (0) */
|
||||
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
|
||||
* `estimatedSrcSize` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_getParams() :
|
||||
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
|
||||
* All fields of `ZSTD_frameParameters` are set to default (0) */
|
||||
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_checkCParams() :
|
||||
* Ensure param values remain within authorized range */
|
||||
@ -463,13 +466,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
* Provides the dictID stored within dictionary.
|
||||
* if @return == 0, the dictionary is not conformant with Zstandard specification.
|
||||
* It can still be loaded, but as a content-only dictionary. */
|
||||
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
|
||||
|
||||
/*! ZSTD_getDictID_fromDDict() :
|
||||
* Provides the dictID of the dictionary loaded into `ddict`.
|
||||
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
||||
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
||||
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
||||
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
||||
|
||||
/*! ZSTD_getDictID_fromFrame() :
|
||||
* Provides the dictID required to decompressed the frame stored within `src`.
|
||||
@ -481,7 +484,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
||||
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
|
||||
* - This is not a Zstandard frame.
|
||||
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
|
||||
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/********************************************************************
|
||||
@ -490,6 +493,7 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
|
||||
@ -602,7 +606,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
||||
This information is not required to properly decode a frame.
|
||||
|
||||
== Special case : skippable frames ==
|
||||
== Special case : skippable frames ==
|
||||
|
||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
|
||||
|
||||
@ -31,7 +31,8 @@
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define MEMMULT 11
|
||||
#define SAMPLESIZE_MAX (128 KB)
|
||||
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
|
||||
static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
|
||||
|
||||
#define NOISELENGTH 32
|
||||
@ -97,7 +98,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
const char* const fileName = fileNamesTable[n];
|
||||
unsigned long long const fs64 = UTIL_getFileSize(fileName);
|
||||
size_t const fileSize = (size_t) MIN(fs64, 128 KB);
|
||||
size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX);
|
||||
if (fileSize > *bufferSizePtr-pos) break;
|
||||
{ FILE* const f = fopen(fileName, "rb");
|
||||
if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
|
||||
@ -163,6 +164,21 @@ static void DiB_saveDict(const char* dictFileName,
|
||||
}
|
||||
|
||||
|
||||
static int g_tooLargeSamples = 0;
|
||||
static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
U64 const cappedFileSize = MIN(fileSize, SAMPLESIZE_MAX);
|
||||
total += cappedFileSize;
|
||||
g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
/*! ZDICT_trainFromBuffer_unsafe() :
|
||||
Strictly Internal use only !!
|
||||
Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
|
||||
@ -181,16 +197,21 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
{
|
||||
void* const dictBuffer = malloc(maxDictSize);
|
||||
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
|
||||
size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
|
||||
size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
|
||||
size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
|
||||
void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
|
||||
int result = 0;
|
||||
|
||||
/* Checks */
|
||||
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
|
||||
g_displayLevel = params.notificationLevel;
|
||||
if (nbFiles < 5) {
|
||||
if (g_tooLargeSamples) {
|
||||
DISPLAYLEVEL(2, "! Warning : some samples are very large \n");
|
||||
DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n");
|
||||
DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX);
|
||||
}
|
||||
if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) {
|
||||
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
|
||||
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
|
||||
DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n");
|
||||
|
||||
3
tests/.gitignore
vendored
3
tests/.gitignore
vendored
@ -3,15 +3,18 @@ fullbench
|
||||
fullbench32
|
||||
fuzzer
|
||||
fuzzer32
|
||||
fuzzer-dll
|
||||
zbufftest
|
||||
zbufftest32
|
||||
zstreamtest
|
||||
zstreamtest32
|
||||
zstreamtest-dll
|
||||
datagen
|
||||
paramgrill
|
||||
paramgrill32
|
||||
roundTripCrash
|
||||
longmatch
|
||||
symbols
|
||||
|
||||
# Tmp test directory
|
||||
zstdtest
|
||||
|
||||
@ -28,12 +28,12 @@ PYTHON ?= python3
|
||||
TESTARTEFACT := versionsTest namespaceTest
|
||||
|
||||
|
||||
CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR)
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
|
||||
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
@ -57,7 +57,7 @@ ZSTREAM_TESTTIME = -T2mn
|
||||
FUZZERTEST= -T5mn
|
||||
ZSTDRTTEST= --test-large-data
|
||||
|
||||
.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest
|
||||
.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
|
||||
|
||||
default: fullbench
|
||||
|
||||
@ -65,6 +65,8 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
|
||||
|
||||
all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
|
||||
|
||||
dll: fuzzer-dll zstreamtest-dll zbufftest-dll
|
||||
|
||||
|
||||
|
||||
zstd:
|
||||
@ -93,28 +95,45 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
|
||||
|
||||
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
|
||||
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest : CFLAGS += -Wno-deprecated-declarations
|
||||
zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) # flag required to silence deprecation warnings
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
|
||||
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
|
||||
@ -127,6 +146,15 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
|
||||
longmatch : $(ZSTD_FILES) longmatch.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
symbols : symbols.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
cp $(ZSTDDIR)/dll/libzstd.dll .
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll
|
||||
else
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
|
||||
endif
|
||||
|
||||
namespaceTest:
|
||||
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
|
||||
$(RM) $@
|
||||
@ -142,8 +170,10 @@ clean:
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fullbench-lib$(EXT) fullbench-dll$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
|
||||
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
|
||||
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
|
||||
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
|
||||
symbols$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
@ -243,4 +273,7 @@ test-zstream32: zstreamtest32
|
||||
test-longmatch: longmatch
|
||||
$(QEMU_SYS) ./longmatch
|
||||
|
||||
test-symbols: symbols
|
||||
$(QEMU_SYS) ./symbols
|
||||
|
||||
endif
|
||||
|
||||
144
tests/symbols.c
Normal file
144
tests/symbols.c
Normal file
@ -0,0 +1,144 @@
|
||||
#include <stdio.h>
|
||||
#include "zstd_errors.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#define ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h"
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#include "zdict.h"
|
||||
|
||||
static const void *symbols[] = {
|
||||
/* zstd.h */
|
||||
&ZSTD_versionNumber,
|
||||
&ZSTD_compress,
|
||||
&ZSTD_decompress,
|
||||
&ZSTD_getDecompressedSize,
|
||||
&ZSTD_maxCLevel,
|
||||
&ZSTD_compressBound,
|
||||
&ZSTD_isError,
|
||||
&ZSTD_getErrorName,
|
||||
&ZSTD_createCCtx,
|
||||
&ZSTD_freeCCtx,
|
||||
&ZSTD_compressCCtx,
|
||||
&ZSTD_createDCtx,
|
||||
&ZSTD_freeDCtx,
|
||||
&ZSTD_decompressDCtx,
|
||||
&ZSTD_compress_usingDict,
|
||||
&ZSTD_decompress_usingDict,
|
||||
&ZSTD_createCDict,
|
||||
&ZSTD_freeCDict,
|
||||
&ZSTD_compress_usingCDict,
|
||||
&ZSTD_createDDict,
|
||||
&ZSTD_freeDDict,
|
||||
&ZSTD_decompress_usingDDict,
|
||||
&ZSTD_createCStream,
|
||||
&ZSTD_freeCStream,
|
||||
&ZSTD_initCStream,
|
||||
&ZSTD_compressStream,
|
||||
&ZSTD_flushStream,
|
||||
&ZSTD_endStream,
|
||||
&ZSTD_CStreamInSize,
|
||||
&ZSTD_CStreamOutSize,
|
||||
&ZSTD_createDStream,
|
||||
&ZSTD_freeDStream,
|
||||
&ZSTD_initDStream,
|
||||
&ZSTD_decompressStream,
|
||||
&ZSTD_DStreamInSize,
|
||||
&ZSTD_DStreamOutSize,
|
||||
/* zstd.h: advanced functions */
|
||||
&ZSTD_estimateCCtxSize,
|
||||
&ZSTD_createCCtx_advanced,
|
||||
&ZSTD_sizeof_CCtx,
|
||||
&ZSTD_createCDict_advanced,
|
||||
&ZSTD_sizeof_CDict,
|
||||
&ZSTD_getCParams,
|
||||
&ZSTD_getParams,
|
||||
&ZSTD_checkCParams,
|
||||
&ZSTD_adjustCParams,
|
||||
&ZSTD_compress_advanced,
|
||||
&ZSTD_isFrame,
|
||||
&ZSTD_estimateDCtxSize,
|
||||
&ZSTD_createDCtx_advanced,
|
||||
&ZSTD_sizeof_DCtx,
|
||||
&ZSTD_sizeof_DDict,
|
||||
&ZSTD_getDictID_fromDict,
|
||||
&ZSTD_getDictID_fromDDict,
|
||||
&ZSTD_getDictID_fromFrame,
|
||||
&ZSTD_createCStream_advanced,
|
||||
&ZSTD_initCStream_srcSize,
|
||||
&ZSTD_initCStream_usingDict,
|
||||
&ZSTD_initCStream_advanced,
|
||||
&ZSTD_initCStream_usingCDict,
|
||||
&ZSTD_resetCStream,
|
||||
&ZSTD_sizeof_CStream,
|
||||
&ZSTD_createDStream_advanced,
|
||||
&ZSTD_initDStream_usingDict,
|
||||
&ZSTD_setDStreamParameter,
|
||||
&ZSTD_initDStream_usingDDict,
|
||||
&ZSTD_resetDStream,
|
||||
&ZSTD_sizeof_DStream,
|
||||
&ZSTD_compressBegin,
|
||||
&ZSTD_compressBegin_usingDict,
|
||||
&ZSTD_compressBegin_advanced,
|
||||
&ZSTD_copyCCtx,
|
||||
&ZSTD_compressContinue,
|
||||
&ZSTD_compressEnd,
|
||||
&ZSTD_getFrameParams,
|
||||
&ZSTD_decompressBegin,
|
||||
&ZSTD_decompressBegin_usingDict,
|
||||
&ZSTD_copyDCtx,
|
||||
&ZSTD_nextSrcSizeToDecompress,
|
||||
&ZSTD_decompressContinue,
|
||||
&ZSTD_nextInputType,
|
||||
&ZSTD_getBlockSizeMax,
|
||||
&ZSTD_compressBlock,
|
||||
&ZSTD_decompressBlock,
|
||||
&ZSTD_insertBlock,
|
||||
/* zstd_errors.h */
|
||||
&ZSTD_getErrorCode,
|
||||
&ZSTD_getErrorString,
|
||||
/* zbuff.h */
|
||||
&ZBUFF_createCCtx,
|
||||
&ZBUFF_freeCCtx,
|
||||
&ZBUFF_compressInit,
|
||||
&ZBUFF_compressInitDictionary,
|
||||
&ZBUFF_compressContinue,
|
||||
&ZBUFF_compressFlush,
|
||||
&ZBUFF_compressEnd,
|
||||
&ZBUFF_createDCtx,
|
||||
&ZBUFF_freeDCtx,
|
||||
&ZBUFF_decompressInit,
|
||||
&ZBUFF_decompressInitDictionary,
|
||||
&ZBUFF_decompressContinue,
|
||||
&ZBUFF_isError,
|
||||
&ZBUFF_getErrorName,
|
||||
&ZBUFF_recommendedCInSize,
|
||||
&ZBUFF_recommendedCOutSize,
|
||||
&ZBUFF_recommendedDInSize,
|
||||
&ZBUFF_recommendedDOutSize,
|
||||
/* zbuff.h: advanced functions */
|
||||
&ZBUFF_createCCtx_advanced,
|
||||
&ZBUFF_createDCtx_advanced,
|
||||
&ZBUFF_compressInit_advanced,
|
||||
/* zdict.h */
|
||||
&ZDICT_trainFromBuffer,
|
||||
&ZDICT_getDictID,
|
||||
&ZDICT_isError,
|
||||
&ZDICT_getErrorName,
|
||||
/* zdict.h: advanced functions */
|
||||
&ZDICT_trainFromBuffer_advanced,
|
||||
&ZDICT_addEntropyTablesFromBuffer,
|
||||
NULL,
|
||||
};
|
||||
|
||||
int main(int argc, const char** argv) {
|
||||
const void **symbol;
|
||||
(void)argc;
|
||||
(void)argv;
|
||||
|
||||
for (symbol = symbols; *symbol != NULL; ++symbol) {
|
||||
printf("%p\n", *symbol);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -28,8 +28,8 @@
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||
#include "zstd.h" /* ZSTD_compressBound */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h" /* ZBUFF_createCCtx_advanced */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
|
||||
#include "zbuff.h" /* ZBUFF_isError */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
@ -265,11 +265,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
BYTE* cNoiseBuffer[5];
|
||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
BYTE* copyBuffer;
|
||||
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
|
||||
size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
|
||||
BYTE* cBuffer;
|
||||
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
|
||||
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
|
||||
BYTE* dstBuffer;
|
||||
size_t dstBufferSize = srcBufferSize;
|
||||
U32 result = 0;
|
||||
|
||||
@ -28,6 +28,7 @@
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */
|
||||
#include "zstd.h" /* ZSTD_compressBound */
|
||||
#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
@ -88,16 +89,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
/*
|
||||
static unsigned FUZ_highbit32(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
for ( ; v32 ; v32>>=1) nbBits++;
|
||||
return nbBits;
|
||||
}
|
||||
*/
|
||||
|
||||
static void* allocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
@ -254,6 +245,42 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* _srcSize compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
|
||||
outBuff.dst = (char*)(compressedBuffer);
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(r)) goto _output_error; }
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
{ unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
|
||||
DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos);
|
||||
DISPLAY("origSize = %u \n", (U32)origSize);
|
||||
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* wrong _srcSize compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
|
||||
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
|
||||
outBuff.dst = (char*)(compressedBuffer);
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(r)) goto _output_error; }
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
|
||||
DISPLAYLEVEL(4, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
|
||||
|
||||
/* Complex context re-use scenario */
|
||||
DISPLAYLEVEL(4, "test%3i : context re-use : ", testNb++);
|
||||
ZSTD_freeCStream(zc);
|
||||
@ -519,7 +546,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
|
||||
U32 n;
|
||||
for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
|
||||
/* compress random chunk into random size dst buffer */
|
||||
/* compress random chunks into randomly sized dst buffers */
|
||||
{ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
|
||||
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
|
||||
|
||||
@ -90,7 +90,7 @@ local int partcompress(FILE *in, z_streamp def)
|
||||
flush = Z_FINISH;
|
||||
LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
ret = deflate(def, flush);
|
||||
LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
LOG_FITBLK("partcompress2 ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
} while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
|
||||
return ret;
|
||||
@ -106,6 +106,7 @@ local int recompress(z_streamp inf, z_streamp def)
|
||||
unsigned char raw[RAWLEN];
|
||||
|
||||
flush = Z_NO_FLUSH;
|
||||
LOG_FITBLK("recompress start\n");
|
||||
do {
|
||||
/* decompress */
|
||||
inf->avail_out = RAWLEN;
|
||||
@ -125,7 +126,7 @@ local int recompress(z_streamp inf, z_streamp def)
|
||||
flush = Z_FINISH;
|
||||
LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
ret = deflate(def, flush);
|
||||
LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
LOG_FITBLK("recompress2deflate ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
} while (ret != Z_STREAM_END && def->avail_out != 0);
|
||||
return ret;
|
||||
@ -165,9 +166,6 @@ int main(int argc, char **argv)
|
||||
ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
|
||||
if (ret != Z_OK || blk == NULL)
|
||||
quit("out of memory");
|
||||
ret = ZWRAP_setPledgedSrcSize(&def, 1<<16);
|
||||
if (ret != Z_OK)
|
||||
quit("ZWRAP_setPledgedSrcSize");
|
||||
|
||||
/* compress from stdin until output full, or no more input */
|
||||
def.avail_out = size + EXCESS;
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
/* minigzip.c -- simulate gzip using the zlib compression library
|
||||
* Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
/*
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
/* gzclose.c -- zlib gzclose() function
|
||||
* Copyright (C) 2004, 2010 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
#include "gzguts.h"
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
|
||||
/* gzguts.h -- zlib internal header definitions for gz* operations
|
||||
* Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
#ifdef _LARGEFILE64_SOURCE
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
/* gzlib.c -- zlib functions common to reading and writing gzip files
|
||||
* Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
#include "gzguts.h"
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
/* gzread.c -- zlib functions for reading gzip files
|
||||
* Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
#include "gzguts.h"
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
/* gzwrite.c -- zlib functions for writing gzip files
|
||||
* Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
#include "gzguts.h"
|
||||
|
||||
@ -301,7 +301,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
||||
}
|
||||
}
|
||||
|
||||
LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||
LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||
if (strm->avail_in > 0) {
|
||||
zwc->inBuffer.src = strm->next_in;
|
||||
zwc->inBuffer.size = strm->avail_in;
|
||||
@ -355,7 +355,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
||||
strm->total_out += zwc->outBuffer.pos;
|
||||
strm->avail_out -= zwc->outBuffer.pos;
|
||||
}
|
||||
LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||
LOG_WRAPPERC("- deflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user