Merge branch 'dev' into doc
This commit is contained in:
Sean Purcell
commit
822c7187d4
18
Makefile
18
Makefile
@ -26,8 +26,7 @@ endif
|
||||
default: lib zstd-release
|
||||
|
||||
.PHONY: all
|
||||
all: allmost
|
||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
|
||||
all: | allmost examples manual
|
||||
|
||||
.PHONY: allmost
|
||||
allmost:
|
||||
@ -68,6 +67,14 @@ zlibwrapper:
|
||||
test:
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: examples
|
||||
examples:
|
||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
|
||||
|
||||
.PHONY: manual
|
||||
manual:
|
||||
$(MAKE) -C contrib/gen_html $@
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
@ -75,6 +82,7 @@ clean:
|
||||
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C examples/ $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
@echo Cleaning completed
|
||||
|
||||
@ -197,16 +205,16 @@ cmaketest:
|
||||
cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||
|
||||
c90test: clean
|
||||
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
|
||||
CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||
|
||||
gnu90test: clean
|
||||
CFLAGS="-std=gnu90" $(MAKE) all
|
||||
CFLAGS="-std=gnu90" $(MAKE) allmost
|
||||
|
||||
c99test: clean
|
||||
CFLAGS="-std=c99" $(MAKE) allmost
|
||||
|
||||
gnu99test: clean
|
||||
CFLAGS="-std=gnu99" $(MAKE) all
|
||||
CFLAGS="-std=gnu99" $(MAKE) allmost
|
||||
|
||||
c11test: clean
|
||||
CFLAGS="-std=c11" $(MAKE) allmost
|
||||
|
||||
2
NEWS
2
NEWS
@ -2,7 +2,7 @@ v1.1.4
|
||||
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
|
||||
cli : new : advanced benchmark command --priority=rt
|
||||
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
||||
API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
|
||||
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
|
||||
API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
|
||||
build:new : meson build system in contrib/meson, by Dima Krasner
|
||||
doc : new : educational decoder, by Sean Purcell
|
||||
|
||||
@ -12,7 +12,7 @@ dependencies:
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-decodecorpus && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi
|
||||
|
||||
@ -7,12 +7,18 @@
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ##########################################################################
|
||||
|
||||
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(CXXFLAGS) $(LDFLAGS)
|
||||
|
||||
ZSTDAPI = ../../lib/zstd.h
|
||||
ZSTDMANUAL = ../../doc/zstd_manual.html
|
||||
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||
LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||
LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
|
||||
LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
||||
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
@ -23,14 +29,23 @@ EXT =
|
||||
endif
|
||||
|
||||
|
||||
.PHONY: default gen_html
|
||||
|
||||
.PHONY: default
|
||||
default: gen_html
|
||||
|
||||
.PHONY: all
|
||||
all: manual
|
||||
|
||||
gen_html: gen_html.cpp
|
||||
$(CXX) $(FLAGS) $^ -o $@$(EXT)
|
||||
$(CXX) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
$(ZSTDMANUAL): gen_html $(ZSTDAPI)
|
||||
echo "Update zstd manual in /doc"
|
||||
./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
|
||||
|
||||
.PHONY: manual
|
||||
manual: gen_html $(ZSTDMANUAL)
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
@$(RM) gen_html$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
@ -17,3 +17,13 @@ It also contains implementations of Huffman and FSE table decoding.
|
||||
|
||||
harness <input-file> <output-file> [dictionary]
|
||||
|
||||
As an additional resource to be used with this decoder,
|
||||
see the `decodecorpus` tool in the [tests] directory.
|
||||
It generates valid Zstandard frames that can be used to verify
|
||||
a Zstandard decoder implementation.
|
||||
Note that to use the tool to verify this decoder implementation,
|
||||
the --content-size flag should be set,
|
||||
as this decoder does not handle streaming decoding,
|
||||
and so it must know the decompressed size in advance.
|
||||
|
||||
[tests]: https://github.com/facebook/zstd/blob/dev/tests/
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
||||
<title>zstd 1.1.2 Manual</title>
|
||||
<title>zstd 1.1.4 Manual</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>zstd 1.1.2 Manual</h1>
|
||||
<h1>zstd 1.1.4 Manual</h1>
|
||||
<hr>
|
||||
<a name="Contents"></a><h2>Contents</h2>
|
||||
<ol>
|
||||
@ -19,13 +19,15 @@
|
||||
<li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
|
||||
<li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
|
||||
<li><a href="#Chapter11">Advanced types</a></li>
|
||||
<li><a href="#Chapter12">Advanced compression functions</a></li>
|
||||
<li><a href="#Chapter13">Advanced decompression functions</a></li>
|
||||
<li><a href="#Chapter14">Advanced streaming functions</a></li>
|
||||
<li><a href="#Chapter15">Buffer-less and synchronous inner streaming functions</a></li>
|
||||
<li><a href="#Chapter16">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter17">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter18">Block functions</a></li>
|
||||
<li><a href="#Chapter12">Compressed size functions</a></li>
|
||||
<li><a href="#Chapter13">Decompressed size functions</a></li>
|
||||
<li><a href="#Chapter14">Advanced compression functions</a></li>
|
||||
<li><a href="#Chapter15">Advanced decompression functions</a></li>
|
||||
<li><a href="#Chapter16">Advanced streaming functions</a></li>
|
||||
<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li>
|
||||
<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter20">Block functions</a></li>
|
||||
</ol>
|
||||
<hr>
|
||||
<a name="Chapter1"></a><h2>Introduction</h2><pre>
|
||||
@ -63,7 +65,7 @@
|
||||
|
||||
<pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
</b><p> `compressedSize` : must be the _exact_ size of a single compressed frame.
|
||||
</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
|
||||
`dstCapacity` is an upper bound of originalSize.
|
||||
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
|
||||
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
||||
@ -71,7 +73,16 @@
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
|
||||
</b><p> 'src' is the start of a zstd compressed frame.
|
||||
</b><p> NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
|
||||
ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
|
||||
frame, but distinguishes empty frames from frames with an unknown size, or errors.
|
||||
|
||||
Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
|
||||
concatenated frames in one buffer, and so is more general.
|
||||
As a result however, it requires more computation and entire frames to be passed to it,
|
||||
as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
|
||||
|
||||
'src' is the start of a zstd compressed frame.
|
||||
@return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
|
||||
note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||
When `return==0`, data to decompress could be any size.
|
||||
@ -88,21 +99,29 @@
|
||||
note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Helper functions</h3><pre><b>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
|
||||
<h3>Helper functions</h3><pre></pre><b><pre>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
|
||||
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
|
||||
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
|
||||
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
|
||||
</b></pre><BR>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
|
||||
|
||||
<h3>Compression context</h3><pre> When compressing many times,
|
||||
it is recommended to allocate a context just once, and re-use it for each successive compression operation.
|
||||
This will make workload friendlier for system's memory.
|
||||
Use one context per thread for parallel execution in multi-threaded environments.
|
||||
</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
|
||||
ZSTD_CCtx* ZSTD_createCCtx(void);
|
||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
|
||||
</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||
<h3>Decompression context</h3><pre></pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||
ZSTD_DCtx* ZSTD_createDCtx(void);
|
||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
</b></pre><BR>
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()).
|
||||
</p></pre><BR>
|
||||
@ -131,11 +150,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
|
||||
<a name="Chapter6"></a><h2>Fast dictionary API</h2><pre></pre>
|
||||
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
||||
</b><p> When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
|
||||
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
||||
ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
|
||||
`dict` can be released after ZSTD_CDict creation.
|
||||
`dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
||||
@ -151,9 +170,9 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
Note that compression level is decided during dictionary creation.
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
|
||||
<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
|
||||
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||
`dict` can be released after creation.
|
||||
dictBuffer can be released after DDict creation, as its content is copied inside DDict
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
|
||||
@ -271,9 +290,9 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
} ZSTD_compressionParameters;
|
||||
</b></pre><BR>
|
||||
<pre><b>typedef struct {
|
||||
unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (if known). */<b>
|
||||
unsigned checksumFlag; </b>/**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */<b>
|
||||
unsigned noDictIDFlag; </b>/**< 1: no dict ID will be saved into frame header (if dictionary compression) */<b>
|
||||
unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (when known) */<b>
|
||||
unsigned checksumFlag; </b>/**< 1: generate a 32-bits checksum at end of frame, for error detection */<b>
|
||||
unsigned noDictIDFlag; </b>/**< 1: no dictID will be saved into frame header (if dictionary compression) */<b>
|
||||
} ZSTD_frameParameters;
|
||||
</b></pre><BR>
|
||||
<pre><b>typedef struct {
|
||||
@ -281,11 +300,56 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
ZSTD_frameParameters fParams;
|
||||
} ZSTD_parameters;
|
||||
</b></pre><BR>
|
||||
<h3>Custom memory allocation functions</h3><pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
</b></pre><BR>
|
||||
<a name="Chapter12"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter12"></a><h2>Compressed size functions</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
|
||||
</b><p> `src` should point to the start of a ZSTD encoded frame
|
||||
`srcSize` must be at least as large as the frame
|
||||
@return : the compressed size of the frame pointed to by `src`, suitable to pass to
|
||||
`ZSTD_decompress` or similar, or an error code if given invalid input.
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter13"></a><h2>Decompressed size functions</h2><pre></pre>
|
||||
|
||||
<pre><b>unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||
</b><p> `src` should point to the start of a ZSTD encoded frame
|
||||
`srcSize` must be at least as large as the frame header. A value greater than or equal
|
||||
to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
|
||||
@return : decompressed size of the frame pointed to be `src` if known, otherwise
|
||||
- ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
||||
- ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small)
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
||||
</b><p> `src` should point the start of a series of ZSTD encoded and/or skippable frames
|
||||
`srcSize` must be the _exact_ size of this series
|
||||
(i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
|
||||
@return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
|
||||
- if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
|
||||
- if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
||||
|
||||
note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||
When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
|
||||
In which case, it's necessary to use streaming mode to decompress data.
|
||||
Optionally, application can still use ZSTD_decompress() while relying on implied limits.
|
||||
(For example, data may be necessarily cut into blocks <= 16 KB).
|
||||
note 2 : decompressed size is always present when compression is done with ZSTD_compress()
|
||||
note 3 : decompressed size can be very large (64-bits value),
|
||||
potentially larger than what local system can handle as a single memory segment.
|
||||
In which case, it's necessary to use streaming mode to decompress data.
|
||||
note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
|
||||
Always ensure result fits within application's authorized limits.
|
||||
Each application can set its own limits.
|
||||
note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
|
||||
read each contained frame header. This is efficient as most of the data is skipped,
|
||||
however it does mean that all frame data must be present and valid.
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
|
||||
</b><p> Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
|
||||
@ -300,7 +364,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
</b><p> Gives the amount of memory used by a given ZSTD_CCtx
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
|
||||
<pre><b>typedef enum {
|
||||
ZSTD_p_forceWindow </b>/* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/<b>
|
||||
} ZSTD_CCtxParameter;
|
||||
</b></pre><BR>
|
||||
<pre><b>size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
|
||||
</b><p> Set advanced parameters, selected through enum ZSTD_CCtxParameter
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError())
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
||||
</b><p> Create a digested dictionary for compression
|
||||
Dictionary content is simply referenced, and therefore stays in dictBuffer.
|
||||
It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
|
||||
ZSTD_parameters params, ZSTD_customMem customMem);
|
||||
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
|
||||
</p></pre><BR>
|
||||
@ -336,7 +415,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter13"></a><h2>Advanced decompression functions</h2><pre></pre>
|
||||
<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>
|
||||
|
||||
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
|
||||
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
|
||||
@ -357,6 +436,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
</b><p> Gives the amount of memory used by a given ZSTD_DCtx
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
|
||||
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||
Dictionary content is simply referenced, and therefore stays in dictBuffer.
|
||||
It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
</b><p> Gives the amount of memory used by a given ZSTD_DDict
|
||||
</p></pre><BR>
|
||||
@ -385,33 +470,33 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code.
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter14"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
|
||||
<h3>Advanced Streaming compression functions</h3><pre><b>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct */<b>
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
||||
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be zero == unknown */<b>
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */<b>
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
||||
</b></pre><BR>
|
||||
<h3>Advanced Streaming decompression functions</h3><pre><b>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
</pre></b><BR>
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
||||
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
|
||||
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
</b></pre><BR>
|
||||
<a name="Chapter15"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
||||
But it's also a complex one, with many restrictions (documented below).
|
||||
Prefer using normal streaming API for an easier experience
|
||||
|
||||
<BR></pre>
|
||||
|
||||
<a name="Chapter16"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
|
||||
<a name="Chapter18"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
|
||||
A ZSTD_CCtx object is required to track streaming operations.
|
||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||
@ -434,20 +519,21 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
In which case, it will "discard" the relevant memory section from its history.
|
||||
|
||||
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
|
||||
It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
|
||||
Without last block mark, frames will be considered unfinished (broken) by decoders.
|
||||
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
|
||||
Without last block mark, frames will be considered unfinished (corrupted) by decoders.
|
||||
|
||||
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
|
||||
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Buffer-less streaming compression functions</h3><pre><b>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b>
|
||||
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
</b></pre><BR>
|
||||
<a name="Chapter17"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
@ -490,7 +576,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
|
||||
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
||||
This information is not required to properly decode a frame.
|
||||
|
||||
== Special case : skippable frames ==
|
||||
== Special case : skippable frames
|
||||
|
||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
|
||||
@ -509,7 +595,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
|
||||
unsigned checksumFlag;
|
||||
} ZSTD_frameParams;
|
||||
</b></pre><BR>
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre><b>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
@ -517,8 +603,8 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
</b></pre><BR>
|
||||
<a name="Chapter18"></a><h2>Block functions</h2><pre>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter20"></a><h2>Block functions</h2><pre>
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
@ -542,10 +628,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
Use ZSTD_insertBlock() in such a case.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Raw zstd block functions</h3><pre><b>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
|
||||
<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
|
||||
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
|
||||
</b></pre><BR>
|
||||
</pre></b><BR>
|
||||
</html>
|
||||
</body>
|
||||
|
||||
@ -369,7 +369,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
||||
totalDstSize += ret;
|
||||
}
|
||||
{
|
||||
size_t const frameSrcSize = ZSTD_getFrameCompressedSize(src, srcSize);
|
||||
size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
|
||||
if (ZSTD_isError(frameSrcSize)) {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
@ -1437,17 +1437,20 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
|
||||
return length;
|
||||
}
|
||||
|
||||
/** ZSTD_getFrameCompressedSize() :
|
||||
/** ZSTD_findFrameCompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* `src` must point to the start of a ZSTD or ZSTD legacy frame
|
||||
* `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
|
||||
* `srcSize` must be at least as large as the frame contained
|
||||
* @return : the compressed size of the frame starting at `src` */
|
||||
size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
|
||||
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
||||
#endif
|
||||
{
|
||||
if (srcSize >= ZSTD_skippableHeaderSize &&
|
||||
(MEM_readLE32(src) & 0xFFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4);
|
||||
} else {
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const ipstart = ip;
|
||||
size_t remainingSize = srcSize;
|
||||
@ -1576,7 +1579,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) {
|
||||
size_t const frameSize = ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
|
||||
size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
||||
size_t decodedSize;
|
||||
if (ZSTD_isError(frameSize)) return frameSize;
|
||||
|
||||
|
||||
@ -123,26 +123,26 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_getFrameCompressedSizeLegacy(const void *src,
|
||||
MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
|
||||
size_t compressedSize)
|
||||
{
|
||||
U32 const version = ZSTD_isLegacy(src, compressedSize);
|
||||
switch(version)
|
||||
{
|
||||
case 1 :
|
||||
return ZSTDv01_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv01_findFrameCompressedSize(src, compressedSize);
|
||||
case 2 :
|
||||
return ZSTDv02_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv02_findFrameCompressedSize(src, compressedSize);
|
||||
case 3 :
|
||||
return ZSTDv03_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv03_findFrameCompressedSize(src, compressedSize);
|
||||
case 4 :
|
||||
return ZSTDv04_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv04_findFrameCompressedSize(src, compressedSize);
|
||||
case 5 :
|
||||
return ZSTDv05_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv05_findFrameCompressedSize(src, compressedSize);
|
||||
case 6 :
|
||||
return ZSTDv06_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv06_findFrameCompressedSize(src, compressedSize);
|
||||
case 7 :
|
||||
return ZSTDv07_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTDv07_findFrameCompressedSize(src, compressedSize);
|
||||
default :
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
|
||||
@ -1992,7 +1992,7 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
|
||||
@ -40,7 +40,7 @@ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
||||
*/
|
||||
size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
||||
|
||||
@ -3378,7 +3378,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
|
||||
static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
@ -3524,9 +3524,9 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
||||
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv02_getFrameCompressedSize(const void *src, size_t compressedSize)
|
||||
size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
|
||||
{
|
||||
return ZSTD_getFrameCompressedSize(src, compressedSize);
|
||||
return ZSTD_findFrameCompressedSize(src, compressedSize);
|
||||
}
|
||||
|
||||
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
||||
|
||||
@ -40,7 +40,7 @@ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
|
||||
*/
|
||||
size_t ZSTDv02_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
||||
|
||||
@ -3019,7 +3019,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
@ -3165,9 +3165,9 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
||||
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_getFrameCompressedSize(src, srcSize);
|
||||
return ZSTD_findFrameCompressedSize(src, srcSize);
|
||||
}
|
||||
|
||||
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
||||
|
||||
@ -40,7 +40,7 @@ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
|
||||
*/
|
||||
size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
|
||||
|
||||
@ -3326,7 +3326,7 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
@ -3782,9 +3782,9 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_getFrameCompressedSize(src, srcSize);
|
||||
return ZSTD_findFrameCompressedSize(src, srcSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
|
||||
|
||||
@ -40,7 +40,7 @@ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
|
||||
*/
|
||||
size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
||||
|
||||
@ -3583,7 +3583,7 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv05_getFrameCompressedSize(const void *src, size_t srcSize)
|
||||
size_t ZSTDv05_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
|
||||
@ -38,7 +38,7 @@ ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv05_isError())
|
||||
*/
|
||||
size_t ZSTDv05_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/* *************************************
|
||||
* Helper functions
|
||||
|
||||
@ -3729,7 +3729,7 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
|
||||
@ -47,7 +47,7 @@ ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv06_isError())
|
||||
*/
|
||||
size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/* *************************************
|
||||
* Helper functions
|
||||
|
||||
@ -3968,7 +3968,7 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
|
||||
@ -54,7 +54,7 @@ ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv07_isError())
|
||||
*/
|
||||
size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/*====== Helper functions ======*/
|
||||
ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||
|
||||
@ -400,12 +400,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||
* Compressed size functions
|
||||
***************************************/
|
||||
|
||||
/*! ZSTD_getFrameCompressedSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame
|
||||
/*! ZSTD_findFrameCompressedSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame or skippable frame
|
||||
* `srcSize` must be at least as large as the frame
|
||||
* @return : the compressed size of the frame pointed to by `src`, suitable to pass to
|
||||
* `ZSTD_decompress` or similar, or an error code if given invalid input. */
|
||||
ZSTDLIB_API size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
/***************************************
|
||||
* Decompressed size functions
|
||||
@ -700,6 +700,9 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||
For skippable frames ZSTD_decompressContinue() always returns 0.
|
||||
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
||||
Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
|
||||
For purposes of decompression, it is valid in both cases to skip the frame using
|
||||
ZSTD_findFrameCompressedSize to find its size in bytes.
|
||||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||
*/
|
||||
|
||||
|
||||
1
tests/.gitignore
vendored
1
tests/.gitignore
vendored
@ -17,6 +17,7 @@ roundTripCrash
|
||||
longmatch
|
||||
symbols
|
||||
legacy
|
||||
decodecorpus
|
||||
pool
|
||||
invalidDictionaries
|
||||
|
||||
|
||||
@ -56,6 +56,7 @@ VOID = /dev/null
|
||||
ZSTREAM_TESTTIME = -T2mn
|
||||
FUZZERTEST ?= -T5mn
|
||||
ZSTDRTTEST = --test-large-data
|
||||
DECODECORPUS_TESTTIME = -T30
|
||||
|
||||
.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
|
||||
|
||||
@ -154,6 +155,9 @@ legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
|
||||
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
decodecorpus : $(filter-out $(ZSTDDIR)/compress/zstd_compress.c, $(wildcard $(ZSTD_FILES))) decodecorpus.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -lm
|
||||
|
||||
symbols : symbols.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
@ -184,7 +188,8 @@ clean:
|
||||
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
|
||||
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
|
||||
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT)
|
||||
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT) \
|
||||
decodecorpus$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
@ -230,7 +235,7 @@ zstd-playTests: datagen
|
||||
file $(ZSTD)
|
||||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
|
||||
ifeq ($(QEMU_SYS),)
|
||||
test: test-pool
|
||||
endif
|
||||
@ -302,6 +307,9 @@ test-symbols: symbols
|
||||
test-legacy: legacy
|
||||
$(QEMU_SYS) ./legacy
|
||||
|
||||
test-decodecorpus: decodecorpus
|
||||
$(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
|
||||
|
||||
test-pool: pool
|
||||
$(QEMU_SYS) ./pool
|
||||
|
||||
|
||||
@ -10,12 +10,14 @@ This directory contains the following programs and scripts:
|
||||
- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
|
||||
- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity
|
||||
- `zstreamtest` : Fuzzer test tool for zstd streaming API
|
||||
- `legacy` : Test tool to test decoding of legacy zstd frames
|
||||
- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
|
||||
|
||||
|
||||
#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
|
||||
|
||||
This script creates `versionsTest` directory to which zstd repository is cloned.
|
||||
Then all taged (released) versions of zstd are compiled.
|
||||
Then all tagged (released) versions of zstd are compiled.
|
||||
In the following step interoperability between zstd versions is checked.
|
||||
|
||||
|
||||
@ -64,3 +66,25 @@ optional arguments:
|
||||
--sleepTime SLEEPTIME
|
||||
frequency of repository checking in seconds
|
||||
```
|
||||
|
||||
#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
|
||||
Command line tool to generate test .zst files.
|
||||
|
||||
This tool will generate .zst files with checksums,
|
||||
as well as optionally output the corresponding correct uncompressed data for
|
||||
extra verfication.
|
||||
|
||||
Example:
|
||||
```
|
||||
./decodecorpus -ptestfiles -otestfiles -n10000 -s5
|
||||
```
|
||||
will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
|
||||
with the zstd checksum field set,
|
||||
as well as the 10,000 original files for more detailed comparison of decompression results.
|
||||
|
||||
```
|
||||
./decodecorpus -t -T1mn
|
||||
```
|
||||
will choose a random seed, and for 1 minute,
|
||||
generate random test frames and ensure that the
|
||||
zstd library correctly decompresses them in both simple and streaming modes.
|
||||
|
||||
1450
tests/decodecorpus.c
Normal file
1450
tests/decodecorpus.c
Normal file
File diff suppressed because it is too large
Load Diff
@ -545,6 +545,13 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* findFrameCompressedSize on skippable frames */
|
||||
DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
|
||||
{ const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
|
||||
size_t const frameSrcSize = 13;
|
||||
if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* error string tests */
|
||||
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
|
||||
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
|
||||
|
||||
@ -16,6 +16,7 @@ static const void *symbols[] = {
|
||||
&ZSTD_decompress,
|
||||
&ZSTD_getDecompressedSize,
|
||||
&ZSTD_findDecompressedSize,
|
||||
&ZSTD_findFrameCompressedSize,
|
||||
&ZSTD_getFrameContentSize,
|
||||
&ZSTD_maxCLevel,
|
||||
&ZSTD_compressBound,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user