added prototypes for advanced parameters for decompression API
required to decode custom formats
This commit is contained in:
Yann Collet
parent
e60f48c549
commit
7c3dea42ce
@ -27,8 +27,8 @@
|
||||
<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li>
|
||||
<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter20">ZSTD_CCtx_params</a></li>
|
||||
<li><a href="#Chapter21">Block functions</a></li>
|
||||
<li><a href="#Chapter20">=== New advanced API (experimental) ===</a></li>
|
||||
<li><a href="#Chapter21">=== Block level API ===</a></li>
|
||||
</ol>
|
||||
<hr>
|
||||
<a name="Chapter1"></a><h2>Introduction</h2><pre>
|
||||
@ -399,7 +399,7 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
||||
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
</b><p> These functions give the current memory usage of selected object.
|
||||
Object memory usage can evolve if it's re-used multiple times.
|
||||
Object memory usage can evolve when re-used multiple times.
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
||||
@ -646,12 +646,12 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
|
||||
@return : 0, or an error code (which can be tested using ZSTD_isError())
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticDCtx() */<b>
|
||||
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b>
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b>
|
||||
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||
@ -783,8 +783,29 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
</pre></b><BR>
|
||||
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||
</b></pre><BR>
|
||||
<h3>New advanced API (experimental, and compression only)</h3><pre></pre><b><pre></pre></b><BR>
|
||||
<a name="Chapter20"></a><h2>=== New advanced API (experimental) ===</h2><pre></pre>
|
||||
|
||||
<pre><b>typedef enum {
|
||||
ZSTD_f_zstd1 = 0, </b>/* Normal zstd frame format, specified in zstd_compression_format.md (default) */<b>
|
||||
ZSTD_f_zstd1_magicless, </b>/* Variant of zstd frame format, without initial 4-bytes magic number.<b>
|
||||
* Useful to save 4 bytes per generated frame.
|
||||
* Decoder will not be able to recognise this format, requiring instructions. */
|
||||
ZSTD_f_zstd1_headerless, </b>/* Variant of zstd frame format, without any frame header;<b>
|
||||
* Other metadata, like block size or frame checksum, are still generated.
|
||||
* Useful to save between 6 and ZSTD_frameHeaderSize_max bytes per generated frame.
|
||||
* However, required decoding parameters will have to be saved or known by some mechanism.
|
||||
* Decoder will not be able to recognise this format, requiring instructions and parameters. */
|
||||
ZSTD_f_zstd1_block </b>/* Generate a zstd compressed block, without any metadata.<b>
|
||||
* Note that size of block content must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB.
|
||||
* See ZSTD_compressBlock() for more details.
|
||||
* Resulting compressed block can be decoded with ZSTD_decompressBlock(). */
|
||||
} ZSTD_format_e;
|
||||
</b></pre><BR>
|
||||
<pre><b>typedef enum {
|
||||
</b>/* compression format */<b>
|
||||
ZSTD_p_format = 10, </b>/* See ZSTD_format_e enum definition.<b>
|
||||
* Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
|
||||
|
||||
</b>/* compression parameters */<b>
|
||||
ZSTD_p_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
|
||||
* Default level is ZSTD_CLEVEL_DEFAULT==3.
|
||||
@ -949,7 +970,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
<pre><b>typedef enum {
|
||||
ZSTD_e_continue=0, </b>/* collect more data, encoder transparently decides when to output result, for optimal conditions */<b>
|
||||
ZSTD_e_flush, </b>/* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */<b>
|
||||
ZSTD_e_end </b>/* flush any remaining data and ends current frame. Any future compression starts a new frame. */<b>
|
||||
ZSTD_e_end </b>/* flush any remaining data and close current frame. Any additional data starts a new frame. */<b>
|
||||
} ZSTD_EndDirective;
|
||||
</b></pre><BR>
|
||||
<pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
@ -959,8 +980,8 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
</b><p> Behave about the same as ZSTD_compressStream. To note :
|
||||
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
|
||||
- Compression parameters cannot be changed once compression is started.
|
||||
- *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
|
||||
- *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
|
||||
- outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
- outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
- @return provides the minimum amount of data still to flush from internal buffers
|
||||
or an error code, which can be tested using ZSTD_isError().
|
||||
if @return != 0, flush is not fully completed, there is some data left within internal buffers.
|
||||
@ -976,6 +997,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
Useful after an error, or to interrupt an ongoing compression job and start a new one.
|
||||
Any internal data not yet flushed is cancelled.
|
||||
Dictionary (if any) is dropped.
|
||||
All parameters are back to default values.
|
||||
It's possible to modify compression parameters after a reset.
|
||||
|
||||
</p></pre><BR>
|
||||
@ -987,26 +1009,30 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
ZSTD_EndDirective endOp);
|
||||
</b><p> Same as ZSTD_compress_generic(),
|
||||
but using only integral types as arguments.
|
||||
Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
|
||||
Argument list is larger than ZSTD_{in,out}Buffer,
|
||||
but can be helpful for binders from dynamic languages
|
||||
which have troubles handling structures containing memory pointers.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter20"></a><h2>ZSTD_CCtx_params</h2><pre>
|
||||
<pre><b>ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
|
||||
</b><p> Quick howto :
|
||||
- ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
|
||||
- ZSTD_CCtxParam_setParameter() : Push parameters one by one into an
|
||||
existing ZSTD_CCtx_params structure. This is similar to
|
||||
ZSTD_CCtx_setParameter().
|
||||
- ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to an existing CCtx. These
|
||||
parameters will be applied to all subsequent compression jobs.
|
||||
- ZSTD_CCtxParam_setParameter() : Push parameters one by one into
|
||||
an existing ZSTD_CCtx_params structure.
|
||||
This is similar to
|
||||
ZSTD_CCtx_setParameter().
|
||||
- ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
|
||||
an existing CCtx.
|
||||
These parameters will be applied to
|
||||
all subsequent compression jobs.
|
||||
- ZSTD_compress_generic() : Do compression using the CCtx.
|
||||
- ZSTD_freeCCtxParams() : Free the memory.
|
||||
|
||||
This can be used with ZSTD_estimateCCtxSize_opaque() for static allocation
|
||||
for single-threaded compression.
|
||||
This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
|
||||
for static allocation for single-threaded compression.
|
||||
|
||||
<BR></pre>
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
|
||||
</b><p> Reset params to default, with the default compression level.
|
||||
@ -1030,22 +1056,84 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
Set one compression parameter, selected by enum ZSTD_cParameter.
|
||||
Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
|
||||
Note : when `value` is an enum, cast it to unsigned for proper type checking.
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
||||
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
|
||||
</b><p> Apply a set of ZSTD_CCtx_params to the compression context.
|
||||
This must be done before the dictionary is loaded.
|
||||
The pledgedSrcSize is treated as unknown.
|
||||
Multithreading parameters are applied only if nbThreads > 1.
|
||||
</b><p> Apply a set of ZSTD_CCtx_params to the compression context.
|
||||
This must be done before the dictionary is loaded.
|
||||
The pledgedSrcSize is treated as unknown.
|
||||
Multithreading parameters are applied only if nbThreads > 1.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter21"></a><h2>Block functions</h2><pre>
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
<h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR>
|
||||
<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
|
||||
</b><p> Create an internal DDict from dict buffer,
|
||||
to be used to decompress next frames.
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
|
||||
meaning "return to no-dictionary mode".
|
||||
Note 1 : `dict` content will be copied internally.
|
||||
Use ZSTD_DCtx_loadDictionary_byReference()
|
||||
to reference dictionary content instead.
|
||||
In which case, the dictionary buffer must outlive its users.
|
||||
Note 2 : Loading a dictionary involves building tables,
|
||||
which has a non-negligible impact on CPU usage and latency.
|
||||
Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
||||
how dictionary content will be interpreted and loaded.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
</b><p> Reference a prepared dictionary, to be used to decompress next frames.
|
||||
The dictionary remains active for decompression of future frames using same DCtx.
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
Note 1 : Currently, only one dictionary can be managed.
|
||||
Referencing a new dictionary effectively "discards" any previous one.
|
||||
Special : adding a NULL DDict means "return to no-dictionary mode".
|
||||
Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);
|
||||
size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);
|
||||
</b><p> Reference a prefix (single-usage dictionary) for next compression job.
|
||||
Prefix is **only used once**. It must be explicitly referenced before each frame.
|
||||
If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
|
||||
Note 2 : Prefix buffer is referenced. It must outlive compression job.
|
||||
Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
||||
Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
|
||||
Note 4 : Referencing a raw content prefix costs almost nothing cpu and memory wise.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
||||
</b><p> Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
|
||||
This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
|
||||
This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
|
||||
By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
|
||||
@return : 0, or an error code (which can be tested using ZSTD_isError()).
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
||||
</b><p> Instruct the decoder context about what kind of data to decode next.
|
||||
This instruction is mandatory to decode data without a fully-formed header,
|
||||
such ZSTD_f_zstd1_magicless for example.
|
||||
@return : 0, or an error code (which can be tested using ZSTD_isError()).
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter21"></a><h2>=== Block level API ===</h2><pre></pre>
|
||||
|
||||
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
@ -1055,7 +1143,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
|
||||
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
|
||||
+ copyCCtx() and copyDCtx() can be used too
|
||||
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
|
||||
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
||||
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
||||
+ For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
|
||||
@ -1066,7 +1154,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
+ In case of multiple successive blocks, should some of them be uncompressed,
|
||||
decoder must be informed of their existence in order to follow proper history.
|
||||
Use ZSTD_insertBlock() for such a case.
|
||||
<BR></pre>
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
|
||||
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
@ -282,6 +282,7 @@ typedef struct {
|
||||
} ZSTD_entropyCTables_t;
|
||||
|
||||
struct ZSTD_CCtx_params_s {
|
||||
ZSTD_format_e format;
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
|
||||
|
||||
@ -256,6 +256,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
|
||||
|
||||
switch(param)
|
||||
{
|
||||
case ZSTD_p_format :
|
||||
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
|
||||
|
||||
case ZSTD_p_compressionLevel:
|
||||
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
|
||||
if (cctx->cdict) return ERROR(stage_wrong);
|
||||
@ -326,6 +329,12 @@ size_t ZSTD_CCtxParam_setParameter(
|
||||
{
|
||||
switch(param)
|
||||
{
|
||||
case ZSTD_p_format :
|
||||
if (value > (unsigned)ZSTD_f_zstd1_block)
|
||||
return ERROR(parameter_unsupported);
|
||||
params->format = (ZSTD_format_e)value;
|
||||
return 0;
|
||||
|
||||
case ZSTD_p_compressionLevel :
|
||||
if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();
|
||||
if (value == 0) return 0;
|
||||
|
||||
160
lib/zstd.h
160
lib/zstd.h
@ -486,7 +486,7 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_sizeof_*() :
|
||||
* These functions give the current memory usage of selected object.
|
||||
* Object memory usage can evolve if it's re-used multiple times. */
|
||||
* Object memory usage can evolve when re-used multiple times. */
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
||||
@ -747,12 +747,12 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
|
||||
|
||||
|
||||
/*===== Advanced Streaming decompression functions =====*/
|
||||
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
|
||||
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */
|
||||
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
|
||||
|
||||
|
||||
@ -908,17 +908,17 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
|
||||
|
||||
|
||||
/*=== New advanced API (experimental, and compression only) ===*/
|
||||
/** === New advanced API (experimental) === **/
|
||||
|
||||
/* notes on API design :
|
||||
* In this proposal, parameters are pushed one by one into an existing CCtx,
|
||||
* In this proposal, parameters are pushed one by one into an existing context,
|
||||
* and then applied on all subsequent compression jobs.
|
||||
* When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
|
||||
*
|
||||
* This API is intended to replace all others experimental API.
|
||||
* It can basically do all other use cases, and even new ones.
|
||||
* It stands a good chance to become "stable",
|
||||
* after a reasonable testing period.
|
||||
* In constrast with _advanced() variants, it stands a reasonable chance to become "stable",
|
||||
* after a testing period.
|
||||
*/
|
||||
|
||||
/* note on naming convention :
|
||||
@ -934,19 +934,25 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
* All enum will be manually set to explicit values before reaching "stable API" status */
|
||||
|
||||
typedef enum {
|
||||
ZSTD_f_zstd1, /* Normal (default) zstd frame format, as specified in zstd_compression_format.md */
|
||||
ZSTD_f_zstd1_magicLess, /* Almost zstd frame format, but without initial 4-bytes magic number */
|
||||
ZSTD_f_zstd1_headerless, /* Almost zstd frame format, but without any frame header;
|
||||
* Other metadata, like block size or frame checksum, are still generated */
|
||||
ZSTD_f_zstd_block /* Pure zstd compressed block, without any metadata.
|
||||
* Note that size of uncompressed block must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB.
|
||||
* See ZSTD_compressBlock() for more details. */
|
||||
} ZSTD_format;
|
||||
ZSTD_f_zstd1 = 0, /* Normal zstd frame format, specified in zstd_compression_format.md (default) */
|
||||
ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number.
|
||||
* Useful to save 4 bytes per generated frame.
|
||||
* Decoder will not be able to recognise this format, requiring instructions. */
|
||||
ZSTD_f_zstd1_headerless, /* Variant of zstd frame format, without any frame header;
|
||||
* Other metadata, like block size or frame checksum, are still generated.
|
||||
* Useful to save between 6 and ZSTD_frameHeaderSize_max bytes per generated frame.
|
||||
* However, required decoding parameters will have to be saved or known by some mechanism.
|
||||
* Decoder will not be able to recognise this format, requiring instructions and parameters. */
|
||||
ZSTD_f_zstd1_block /* Generate a zstd compressed block, without any metadata.
|
||||
* Note that size of block content must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB.
|
||||
* See ZSTD_compressBlock() for more details.
|
||||
* Resulting compressed block can be decoded with ZSTD_decompressBlock(). */
|
||||
} ZSTD_format_e;
|
||||
|
||||
typedef enum {
|
||||
/* compression format */
|
||||
ZSTD_p_format = 10, /* See ZSTD_format enum definition.
|
||||
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. */
|
||||
ZSTD_p_format = 10, /* See ZSTD_format_e enum definition.
|
||||
* Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
|
||||
|
||||
/* compression parameters */
|
||||
ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
|
||||
@ -1116,15 +1122,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
||||
typedef enum {
|
||||
ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
|
||||
ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
|
||||
ZSTD_e_end /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
|
||||
ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */
|
||||
} ZSTD_EndDirective;
|
||||
|
||||
/*! ZSTD_compress_generic() :
|
||||
* Behave about the same as ZSTD_compressStream. To note :
|
||||
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
|
||||
* - Compression parameters cannot be changed once compression is started.
|
||||
* - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
|
||||
* - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
|
||||
* - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
* - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
* - @return provides the minimum amount of data still to flush from internal buffers
|
||||
* or an error code, which can be tested using ZSTD_isError().
|
||||
* if @return != 0, flush is not fully completed, there is some data left within internal buffers.
|
||||
@ -1143,6 +1149,7 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
* Useful after an error, or to interrupt an ongoing compression job and start a new one.
|
||||
* Any internal data not yet flushed is cancelled.
|
||||
* Dictionary (if any) is dropped.
|
||||
* All parameters are back to default values.
|
||||
* It's possible to modify compression parameters after a reset.
|
||||
*/
|
||||
ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
|
||||
@ -1151,7 +1158,7 @@ ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
|
||||
/*! ZSTD_compress_generic_simpleArgs() :
|
||||
* Same as ZSTD_compress_generic(),
|
||||
* but using only integral types as arguments.
|
||||
* Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
|
||||
* Argument list is larger than ZSTD_{in,out}Buffer,
|
||||
* but can be helpful for binders from dynamic languages
|
||||
* which have troubles handling structures containing memory pointers.
|
||||
*/
|
||||
@ -1162,19 +1169,22 @@ size_t ZSTD_compress_generic_simpleArgs (
|
||||
ZSTD_EndDirective endOp);
|
||||
|
||||
|
||||
/** ZSTD_CCtx_params
|
||||
*
|
||||
/*! ZSTD_CCtx_params :
|
||||
* Quick howto :
|
||||
* - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
|
||||
* - ZSTD_CCtxParam_setParameter() : Push parameters one by one into an
|
||||
* existing ZSTD_CCtx_params structure. This is similar to
|
||||
* ZSTD_CCtx_setParameter().
|
||||
* - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to an existing CCtx. These
|
||||
* parameters will be applied to all subsequent compression jobs.
|
||||
* - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
|
||||
* an existing ZSTD_CCtx_params structure.
|
||||
* This is similar to
|
||||
* ZSTD_CCtx_setParameter().
|
||||
* - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
|
||||
* an existing CCtx.
|
||||
* These parameters will be applied to
|
||||
* all subsequent compression jobs.
|
||||
* - ZSTD_compress_generic() : Do compression using the CCtx.
|
||||
* - ZSTD_freeCCtxParams() : Free the memory.
|
||||
*
|
||||
* This can be used with ZSTD_estimateCCtxSize_opaque() for static allocation
|
||||
* for single-threaded compression.
|
||||
* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
|
||||
* for static allocation for single-threaded compression.
|
||||
*/
|
||||
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
|
||||
|
||||
@ -1202,22 +1212,96 @@ ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
|
||||
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
||||
* Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
|
||||
* Note : when `value` is an enum, cast it to unsigned for proper type checking.
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
|
||||
|
||||
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
|
||||
* Apply a set of ZSTD_CCtx_params to the compression context.
|
||||
* This must be done before the dictionary is loaded.
|
||||
* The pledgedSrcSize is treated as unknown.
|
||||
* Multithreading parameters are applied only if nbThreads > 1.
|
||||
* Apply a set of ZSTD_CCtx_params to the compression context.
|
||||
* This must be done before the dictionary is loaded.
|
||||
* The pledgedSrcSize is treated as unknown.
|
||||
* Multithreading parameters are applied only if nbThreads > 1.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
||||
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
|
||||
|
||||
/**
|
||||
Block functions
|
||||
|
||||
/*=== Advanced parameters for decompression API ===*/
|
||||
|
||||
/* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object,
|
||||
* but before starting decompression of a frame.
|
||||
*/
|
||||
|
||||
/*! ZSTD_DCtx_loadDictionary() :
|
||||
* Create an internal DDict from dict buffer,
|
||||
* to be used to decompress next frames.
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
|
||||
* meaning "return to no-dictionary mode".
|
||||
* Note 1 : `dict` content will be copied internally.
|
||||
* Use ZSTD_DCtx_loadDictionary_byReference()
|
||||
* to reference dictionary content instead.
|
||||
* In which case, the dictionary buffer must outlive its users.
|
||||
* Note 2 : Loading a dictionary involves building tables,
|
||||
* which has a non-negligible impact on CPU usage and latency.
|
||||
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
||||
* how dictionary content will be interpreted and loaded.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
|
||||
|
||||
|
||||
/*! ZSTD_DCtx_refDDict() :
|
||||
* Reference a prepared dictionary, to be used to decompress next frames.
|
||||
* The dictionary remains active for decompression of future frames using same DCtx.
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
* Note 1 : Currently, only one dictionary can be managed.
|
||||
* Referencing a new dictionary effectively "discards" any previous one.
|
||||
* Special : adding a NULL DDict means "return to no-dictionary mode".
|
||||
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
|
||||
/*! ZSTD_DCtx_refPrefix() :
|
||||
* Reference a prefix (single-usage dictionary) for next compression job.
|
||||
* Prefix is **only used once**. It must be explicitly referenced before each frame.
|
||||
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
* Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
|
||||
* Note 2 : Prefix buffer is referenced. It must outlive compression job.
|
||||
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
||||
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
|
||||
* Note 4 : Referencing a raw content prefix costs almost nothing cpu and memory wise.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);
|
||||
|
||||
|
||||
/*! ZSTD_DCtx_setMaxWindowSize() :
|
||||
* Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
|
||||
* This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
|
||||
* This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
|
||||
* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
||||
|
||||
|
||||
/*! ZSTD_DCtx_setFormat() :
|
||||
* Instruct the decoder context about what kind of data to decode next.
|
||||
* This instruction is mandatory to decode data without a fully-formed header,
|
||||
* such ZSTD_f_zstd1_magicless for example.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
||||
|
||||
|
||||
|
||||
/** === Block level API === **/
|
||||
|
||||
/*!
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user