diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c
index 49f1d4d5..524d1378 100644
--- a/contrib/adaptive-compression/adapt.c
+++ b/contrib/adaptive-compression/adapt.c
@@ -579,7 +579,7 @@ static void* compressionThread(void* arg)
params.cParams.windowLog = 23;
{
size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0);
- size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1);
+ size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_c_forceMaxWindow, 1);
if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {
DISPLAY("Error: something went wrong while starting compression\n");
signalErrorToThreads(ctx);
diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c
index 4ab666bd..59746665 100644
--- a/contrib/seekable_format/zstdseek_compress.c
+++ b/contrib/seekable_format/zstdseek_compress.c
@@ -268,7 +268,7 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
static inline size_t ZSTD_seekable_seekTableSize(const ZSTD_frameLog* fl)
{
size_t const sizePerFrame = 8 + (fl->checksumFlag?4:0);
- size_t const seekTableLen = ZSTD_skippableHeaderSize +
+ size_t const seekTableLen = ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->size +
ZSTD_seekTableFooterSize;
@@ -307,24 +307,24 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
size_t const seekTableLen = ZSTD_seekable_seekTableSize(fl);
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0));
- CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_skippableHeaderSize,
+ CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_SKIPPABLEHEADERSIZE,
4));
while (fl->seekTableIndex < fl->size) {
CHECK_Z(ZSTD_stwrite32(fl, output,
fl->entries[fl->seekTableIndex].cSize,
- ZSTD_skippableHeaderSize +
+ ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->seekTableIndex + 0));
CHECK_Z(ZSTD_stwrite32(fl, output,
fl->entries[fl->seekTableIndex].dSize,
- ZSTD_skippableHeaderSize +
+ ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->seekTableIndex + 4));
if (fl->checksumFlag) {
CHECK_Z(ZSTD_stwrite32(
fl, output, fl->entries[fl->seekTableIndex].checksum,
- ZSTD_skippableHeaderSize +
+ ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->seekTableIndex + 8));
}
diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c
index b4c48754..54f0a084 100644
--- a/contrib/seekable_format/zstdseek_decompress.c
+++ b/contrib/seekable_format/zstdseek_decompress.c
@@ -275,7 +275,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
{ U32 const numFrames = MEM_readLE32(zs->inBuff);
U32 const sizePerEntry = 8 + (checksumFlag?4:0);
U32 const tableSize = sizePerEntry * numFrames;
- U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize;
+ U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
{
@@ -290,7 +290,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
if (MEM_readLE32(zs->inBuff) != (ZSTD_MAGIC_SKIPPABLE_START | 0xE)) {
return ERROR(prefix_unknown);
}
- if (MEM_readLE32(zs->inBuff+4) + ZSTD_skippableHeaderSize != frameSize) {
+ if (MEM_readLE32(zs->inBuff+4) + ZSTD_SKIPPABLEHEADERSIZE != frameSize) {
return ERROR(prefix_unknown);
}
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 107e3947..92566e1d 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -19,16 +19,19 @@
Streaming compression - HowTo
Streaming decompression - HowTo
ADVANCED AND EXPERIMENTAL FUNCTIONS
-Frame size functions
-Memory management
-Advanced compression functions
-Advanced decompression functions
-Advanced streaming functions
-Buffer-less and synchronous inner streaming functions
-Buffer-less streaming compression (synchronous mode)
-Buffer-less streaming decompression (synchronous mode)
-New advanced API (experimental)
-Block level API
+Candidate API for promotion to stable status
+Advanced compression API
+experimental API (static linking only)
+Frame size functions
+Memory management
+Advanced compression functions
+Advanced decompression functions
+Advanced streaming functions
+Buffer-less and synchronous inner streaming functions
+Buffer-less streaming compression (synchronous mode)
+Buffer-less streaming decompression (synchronous mode)
+ZSTD_getFrameHeader() :
+Block level API
Introduction
@@ -64,7 +67,7 @@
Version
-unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
+unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
Default constant
@@ -139,11 +142,13 @@ int ZSTD_maxCLevel(void); /*!< maximum compression lev
ZSTD_CCtx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
+size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
- Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
+
Same as ZSTD_compress(), using an explicit ZSTD_CCtx
+ The function will compress at requested compression level,
+ ignoring any other parameter
Decompression context
When decompressing many times,
@@ -155,10 +160,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
- Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx())
+
Same as ZSTD_decompress(),
+ requires an allocated ZSTD_DCtx.
+ Compatible with sticky parameters.
+
Simple dictionary API
@@ -168,18 +176,22 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
int compressionLevel);
- Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+
Compression at an explicit compression level using a Dictionary.
+ A dictionary can be any arbitrary data segment (also called a prefix),
+ or a buffer with specified information (see dictBuilder/zdict.h).
Note : This function loads the dictionary, resulting in significant startup delay.
- Note : When `dict == NULL || dictSize < 8` no dictionary is used.
+ It's intended for a dictionary used only once.
+ Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used.
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
- Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+
Decompression using a known Dictionary.
Dictionary must be identical to the one used during compression.
Note : This function loads the dictionary, resulting in significant startup delay.
+ It's intended for a dictionary used only once.
Note : When `dict == NULL || dictSize < 8` no dictionary is used.
@@ -187,11 +199,12 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
- When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+
When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
+ ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict
- Note : A ZSTD_CDict can be created with an empty dictionary, but it is inefficient for small data.
+ `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
+ Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data.
size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -203,16 +216,14 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
const void* src, size_t srcSize,
const ZSTD_CDict* cdict);
Compression using a digested Dictionary.
- Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
- Note that compression level is decided during dictionary creation.
- Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no)
- Note : ZSTD_compress_usingCDict() can be used with a ZSTD_CDict created from an empty dictionary.
- But it is inefficient for small data, and it is recommended to use ZSTD_compressCCtx().
+ Recommended when same dictionary is used multiple times.
+ Note : compression level is _decided at dictionary creation time_,
+ and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no)
ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
Create a digested dictionary, ready to start decompression operation without startup delay.
- dictBuffer can be released after DDict creation, as its content is copied inside DDict
+ dictBuffer can be released after DDict creation, as its content is copied inside DDict.
size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
@@ -224,7 +235,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
const void* src, size_t srcSize,
const ZSTD_DDict* ddict);
Decompression using a digested Dictionary.
- Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times.
+ Recommended when same dictionary is used multiple times.
Streaming
@@ -245,13 +256,17 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
A ZSTD_CStream object is required to track streaming operation.
Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
- It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
- since it will play nicer with system's memory, by re-using already allocated memory.
- Use one separate ZSTD_CStream per thread for parallel execution.
+ It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
- Start a new compression by initializing ZSTD_CStream context.
- Use ZSTD_initCStream() to start a new compression operation.
- Use variants ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for streaming with dictionary (experimental section)
+ For parallel execution, use one separate ZSTD_CStream per thread.
+
+ note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+
+ Parameters are sticky : when starting a new compression on the same context,
+ it will re-use the same sticky parameters as previous compression session.
+ When in doubt, it's recommended to fully initialize the context before usage.
+ Use ZSTD_initCStream() to set the parameter to a selected compression level.
+ Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
Use ZSTD_compressStream() as many times as necessary to consume input stream.
The function will automatically update both `pos` fields within `input` and `output`.
@@ -260,12 +275,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
in which case `input.pos < input.size`.
The caller must check if input has been entirely consumed.
If not, the caller must make some room to receive more compressed data,
- typically by emptying output buffer, or allocating a new output buffer,
and then present again remaining input data.
- @return : a size hint, preferred nb of bytes to use as input for next function call
- or an error code, which can be tested using ZSTD_isError().
- Note 1 : it's just a hint, to help latency a little, any other value will work fine.
- Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
+ @return : a size hint, preferred nb of bytes to use as input for next function call
+ or an error code, which can be tested using ZSTD_isError().
+ Note 1 : it's just a hint, to help latency a little, any value will work fine.
+ Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
using ZSTD_flushStream(). `output->pos` will be updated.
@@ -305,25 +319,24 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
ZSTD_DStream objects can be re-used multiple times.
- Use ZSTD_initDStream() to start a new decompression operation,
- or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
- @return : recommended first input size
+ Use ZSTD_initDStream() to start a new decompression operation.
+ @return : recommended first input size
+ Alternatively, use advanced API to set specific properties.
Use ZSTD_decompressStream() repetitively to consume your input.
The function will update both `pos` fields.
If `input.pos < input.size`, some input has not been consumed.
It's up to the caller to present again remaining data.
- The function tries to flush all data decoded immediately, repecting buffer sizes.
+ The function tries to flush all data decoded immediately, respecting output buffer size.
If `output.pos < output.size`, decoder has flushed everything it could.
- But if `output.pos == output.size`, there is no such guarantee,
- it's likely that some decoded data was not flushed and still remains within internal buffers.
+ But if `output.pos == output.size`, there might be some data left within internal buffers.,
In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
- When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+ Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
@return : 0 when a frame is completely decoded and fully flushed,
or an error code, which can be tested using ZSTD_isError(),
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
- the return value is a suggested next input size (a hint for better latency)
- that will never load more than the current frame.
+ the return value is a suggested next input size (just a hint for better latency)
+ that will never request more than the remaining frame size.
@@ -340,32 +353,465 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
ADVANCED AND EXPERIMENTAL FUNCTIONS
- The definitions in this section are considered experimental.
- They should never be used with a dynamic library, as prototypes may change in the future.
+ The definitions in the following section are considered experimental.
They are provided for advanced scenarios.
+ They should never be used with a dynamic library, as prototypes may change in the future.
Use them only in association with static linking.
+Candidate API for promotion to stable status
+ The following symbols and constants form the "staging area" :
+ they are considered to join "stable API" by v1.4.0.
+ The proposal is written so that it can be made stable "as is",
+ though it's still possible to suggest improvements.
+ Staging is in fact last chance for changes,
+ the API is locked once reaching "stable" status.
+
+
+
int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */
-typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
- ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
+size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+ `src` should point to the start of a ZSTD frame or skippable frame.
+ `srcSize` must be >= first frame size
+ @return : the compressed size of the first frame starting at `src`,
+ suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ or an error code if input is invalid
+
+
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+ These functions give the _current_ memory usage of selected object.
+ Note that object memory usage can evolve (increase or decrease) over time.
+
+
+Advanced compression API
+
+typedef enum { ZSTD_fast=1,
+ ZSTD_dfast=2,
+ ZSTD_greedy=3,
+ ZSTD_lazy=4,
+ ZSTD_lazy2=5,
+ ZSTD_btlazy2=6,
+ ZSTD_btopt=7,
+ ZSTD_btultra=8
+ /* note : new strategies might be added in the future.
+ Only the order (from fast to strong) is guaranteed, not the exact position.
+ new strategy names might be introduced, pushing the maximum number upward */
+} ZSTD_strategy;
+
+typedef enum {
+
+ /* compression parameters */
+ ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
+ * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+ * Note 1 : it's possible to pass a negative compression level.
+ * Note 2 : setting a level sets all default values of other compression parameters */
+ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+ * Special: value 0 means "use default windowLog".
+ * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+ * requires explicitly allowing such window size at decompression stage if using streaming. */
+ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
+ * Resulting memory usage is (1 << (hashLog+2)).
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+ * Larger tables improve compression ratio of strategies <= dFast,
+ * and improve speed of strategies > dFast.
+ * Special: value 0 means "use default hashLog". */
+ ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
+ * Resulting memory usage is (1 << (chainLog+2)).
+ * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+ * Larger tables result in better and slower compression.
+ * This parameter is useless when using "fast" strategy.
+ * It's still useful when using "dfast" strategy,
+ * in which case it defines a secondary probe table.
+ * Special: value 0 means "use default chainLog". */
+ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
+ * More attempts result in better and slower compression.
+ * This parameter is useless when using "fast" and "dFast" strategies.
+ * Special: value 0 means "use default searchLog". */
+ ZSTD_c_minMatch=105, /* Minimum size of searched matches.
+ * Note that Zstandard can still find matches of smaller size,
+ * it just tweaks its search algorithm to look for this size and larger.
+ * Larger values increase compression and decompression speed, but decrease ratio.
+ * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
+ * , for all strategies > fast, effective maximum is 6.
+ * Special: value 0 means "use default minMatchLength". */
+ ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
+ * For strategies btopt & btultra:
+ * Length of Match considered "good enough" to stop search.
+ * Larger values make compression stronger, and slower.
+ * For strategy fast:
+ * Distance between match sampling.
+ * Larger values make compression faster, and weaker.
+ * Special: value 0 means "use default targetLength". */
+ ZSTD_c_compressionStrategy=107, /* See ZSTD_strategy enum definition.
+ * The higher the value of selected strategy, the more complex it is,
+ * resulting in stronger and slower compression.
+ * Special: value 0 means "use default strategy". */
+
+ /* LDM mode parameters */
+ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
+ * This parameter is designed to improve compression ratio
+ * for large inputs, by finding large matches at long distance.
+ * It increases memory usage and window size.
+ * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+ * except when expressly set to a different value. */
+ ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
+ * Larger values increase memory usage and compression ratio,
+ * but decrease compression speed.
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+ * default: windowlog - 7.
+ * Special: value 0 means "automatically determine hashlog". */
+ ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
+ * Larger/too small values usually decrease compression ratio.
+ * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+ * Special: value 0 means "use default value" (default: 64). */
+ ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
+ * Larger values improve collision resolution but decrease compression speed.
+ * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+ * Special: value 0 means "use default value" (default: 3). */
+ ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
+ * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+ * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+ * Larger values improve compression speed.
+ * Deviating far from default value will likely result in a compression ratio decrease.
+ * Special: value 0 means "automatically determine hashRateLog". */
+
+ /* frame parameters */
+ ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+ * Content size must be known at the beginning of compression,
+ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
+ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
+ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
+
+ /* multi-threading parameters */
+ /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+ * They return an error otherwise. */
+ ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
+ * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
+ * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+ * while compression work is performed in parallel, within worker threads.
+ * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+ * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+ * More workers improve speed, but also increase memory usage.
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
+ ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
+ * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+ * 0 means default, which is dynamically determined based on compression parameters.
+ * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
+ * The minimum size is automatically and transparently enforced */
+ ZSTD_c_overlapSizeLog=402, /* Size of previous job reloaded at the beginning of each job, as a fraction of window size.
+ * This value is enforced only when nbWorkers >= 1.
+ * Larger values increase compression ratio, but decrease speed.
+ * Values range from 0 (no overlap) to 9 (overlap a full windowSize).
+ * Each rank (except 0) increase/decrease load size by a factor 2
+ * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:w/256;
+ * default value is 6 : use 1/8th of windowSize */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_c_rsyncable
+ * ZSTD_c_format
+ * ZSTD_c_forceMaxWindow
+ * ZSTD_c_forceAttachDict
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly
+ */
+ ZSTD_c_experimentalParam1=500,
+ ZSTD_c_experimentalParam2=10,
+ ZSTD_c_experimentalParam3=1000,
+ ZSTD_c_experimentalParam4
+} ZSTD_cParameter;
typedef struct {
- unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
- unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
- unsigned hashLog; /**< dispatch table : larger == faster, more memory */
- unsigned searchLog; /**< nb of searches : larger == more compression, slower */
- unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
- unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
- ZSTD_strategy strategy;
+ size_t error;
+ int lowerBound;
+ int upperBound;
+} ZSTD_bounds;
+
+ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+ All parameters must belong to an interval with lower and upper bounds,
+ otherwise they will either trigger an error or be automatically clamped.
+ @return : a structure, ZSTD_bounds, which contains
+ - an error status field, which must be tested using ZSTD_isError()
+ - lower and upper bounds, both inclusive
+
+
+
+size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+ Set one compression parameter, selected by enum ZSTD_cParameter.
+ All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+ Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ Setting a parameter is generally only possible during frame initialization (before starting compression).
+ Exception : when using multi-threading mode (nbWorkers >= 1),
+ the following parameters can be updated _during_ compression (within same frame):
+ => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+ new parameters will be active for next job only (after a flush()).
+ @return : an error code (which can be tested using ZSTD_isError()).
+
+
+
+size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+ Total input data size to be compressed as a single frame.
+ This value will be controlled at end of frame, and trigger an error if not respected.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+ In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+ Note 2 : pledgedSrcSize is only valid once, for the next frame.
+ It's discarded at the end of the frame.
+ Note 3 : If all data is provided and consumed in a single round,
+ this value is automatically overriden by srcSize instead.
+
+
+
+size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+ Create an internal CDict from `dict` buffer.
+ Decompression will have to use same dictionary.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ meaning "return to no-dictionary mode".
+ Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
+ To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ Note 2 : Loading a dictionary involves building tables.
+ It's also a CPU consuming operation, with non-negligible impact on latency.
+ Tables are dependent on compression parameters, and for this reason,
+ compression parameters can no longer be changed after loading a dictionary.
+ Note 3 :`dict` content will be copied internally.
+ Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ In such a case, dictionary buffer must outlive its users.
+ Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ to precisely select how dictionary content must be interpreted.
+
+
+size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+ Reference a prepared dictionary, to be used for all next compressed frames.
+ Note that compression parameters are enforced from within CDict,
+ and supercede any compression parameter previously set within CCtx.
+ The dictionary will remain valid for future compressed frames using same CCtx.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ Note 1 : Currently, only one dictionary can be managed.
+ Referencing a new dictionary effectively "discards" any previous one.
+ Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx.
+
+
+size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+ const void* prefix, size_t prefixSize);
+ Reference a prefix (single-usage dictionary) for next compressed frame.
+ A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ Decompression will need same prefix to properly regenerate data.
+ Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ Its content must remain unmodified during compression.
+ Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ ensure that the window size is large enough to contain the entire source.
+ See ZSTD_c_windowLog.
+ Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ It's a CPU consuming operation, with non-negligible impact on latency.
+ If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+ Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation.
+
+
+typedef enum {
+ ZSTD_reset_session_only = 1,
+ ZSTD_reset_parameters = 2,
+ ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+
+size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+ There are 2 different things that can be reset, independently or jointly :
+ - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+ Useful after an error, or to interrupt any ongoing compression.
+ Any internal data not yet flushed is cancelled.
+ Compression parameters and dictionary remain unchanged.
+ They will be used to compress next frame.
+ Resetting session never fails.
+ - The parameters : changes all parameters back to "default".
+ This removes any reference to any dictionary too.
+ Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+ otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+ - Both : similar to resetting the session, followed by resetting parameters.
+
+
+
+size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize);
+ Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ - The function is always blocking, returns when compression is completed.
+ Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ @return : compressed size written into `dst` (<= `dstCapacity),
+ or an error code if it fails (which can be tested using ZSTD_isError()).
+
+
+
+typedef enum {
+ ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+ ZSTD_e_flush=1, /* flush any data provided so far,
+ * it creates (at least) one new block, that can be decoded immediately on reception;
+ * frame will continue: any future data can still reference previously compressed data, improving compression. */
+ ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
+ * note that frame is only closed after compressed data is fully flushed (return value == 0).
+ * After that point, any additional data starts a new frame.
+ * note : each frame is independent (does not reference any content from previous frame). */
+} ZSTD_EndDirective;
+
+size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp);
+ Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
+ - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
+ and then immediately returns, just indicating that there is some data remaining to be flushed.
+ The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ or an error code, which can be tested using ZSTD_isError().
+ if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ Before starting a new compression job, or changing compression parameters,
+ it is required to fully flush internal buffers.
+
+
+
+typedef enum {
+
+ ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+ * the streaming API will refuse to allocate memory buffer
+ * in order to protect the host from unreasonable memory requirements.
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_c_format
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly
+ */
+ ZSTD_d_experimentalParam1=1000
+
+} ZSTD_dParameter;
+
+ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+ All parameters must belong to an interval with lower and upper bounds,
+ otherwise they will either trigger an error or be automatically clamped.
+ @return : a structure, ZSTD_bounds, which contains
+ - an error status field, which must be tested using ZSTD_isError()
+ - both lower and upper bounds, inclusive
+
+
+
+size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+ Set one compression parameter, selected by enum ZSTD_dParameter.
+ All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ Setting a parameter is only possible during frame initialization (before starting decompression).
+ @return : 0, or an error code (which can be tested using ZSTD_isError()).
+
+
+
+size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ Create an internal DDict from dict buffer,
+ to be used to decompress next frames.
+ The dictionary remains valid for all future frames, until explicitly invalidated.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ meaning "return to no-dictionary mode".
+ Note 1 : Loading a dictionary involves building tables,
+ which has a non-negligible impact on CPU usage and latency.
+ It's recommended to "load once, use many times", to amortize the cost
+ Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ how dictionary content is loaded and interpreted.
+
+
+
+size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+ Reference a prepared dictionary, to be used to decompress next frames.
+ The dictionary remains active for decompression of future frames using same DCtx.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Note 1 : Currently, only one dictionary can be managed.
+ Referencing a new dictionary effectively "discards" any previous one.
+ Special: referencing a NULL DDict means "return to no-dictionary mode".
+ Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+
+
+
+size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+ const void* prefix, size_t prefixSize);
+ Reference a prefix (single-usage dictionary) to decompress next frame.
+ This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ and must use the same prefix as the one used during compression.
+ Prefix is **only used once**. Reference is discarded at end of frame.
+ End of frame is reached when ZSTD_decompressStream() returns 0.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ Prefix buffer must remain unmodified up to the end of frame,
+ reached when ZSTD_decompressStream() returns 0.
+ Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ A full dictionary is more costly, as it requires building tables.
+
+
+
+size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+ Return a DCtx to clean state.
+ Session and parameters can be reset jointly or separately.
+ Parameters can only be reset when no active frame is being decompressed.
+ @return : 0, or an error code, which can be tested with ZSTD_isError()
+
+
+
+experimental API (static linking only)
+ The following symbols and constants
+ are not planned to join "stable API" status in the near future.
+ They can still change in future versions.
+ Some of them are planned to remain in the static_only section indefinitely.
+ Some of them might be removed in the future (especially when redundant with existing stable functions)
+
+
+
+typedef struct {
+ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
+ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
+ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
+ unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
+ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+ ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
} ZSTD_compressionParameters;
typedef struct {
- unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
- unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */
- unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */
+ int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+ int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
+ int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@@ -374,25 +820,65 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} ZSTD_parameters;
typedef enum {
- ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
- ZSTD_dct_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
- ZSTD_dct_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
+ ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
+ ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+ ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
} ZSTD_dictContentType_e;
typedef enum {
- ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
- ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
+ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
+ ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
} ZSTD_dictLoadMethod_e;
-Frame size functions
-
-size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
- `src` should point to the start of a ZSTD encoded frame or skippable frame
- `srcSize` must be >= first frame size
- @return : the compressed size of the first frame starting at `src`,
- suitable to pass to `ZSTD_decompress` or similar,
- or an error code if input is invalid
-
+typedef enum {
+ /* Opened question : should we have a format ZSTD_f_auto ?
+ * Today, it would mean exactly the same as ZSTD_f_zstd1.
+ * But, in the future, should several formats become supported,
+ * on the compression side, it would mean "default format".
+ * On the decompression side, it would mean "automatic format detection",
+ * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
+ * Since meaning is a little different, another option could be to define different enums for compression and decompression.
+ * This question could be kept for later, when there are actually multiple formats to support,
+ * but there is also the question of pinning enum values, and pinning value `0` is especially important */
+ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
+ ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
+ * Useful to save 4 bytes per generated frame.
+ * Decoder cannot recognise automatically this format, requiring this instruction. */
+} ZSTD_format_e;
+
+typedef enum {
+ /* Note: this enum and the behavior it controls are effectively internal
+ * implementation details of the compressor. They are expected to continue
+ * to evolve and should be considered only in the context of extremely
+ * advanced performance tuning.
+ *
+ * Zstd currently supports the use of a CDict in two ways:
+ *
+ * - The contents of the CDict can be copied into the working context. This
+ * means that the compression can search both the dictionary and input
+ * while operating on a single set of internal tables. This makes
+ * the compression faster per-byte of input. However, the initial copy of
+ * the CDict's tables incurs a fixed cost at the beginning of the
+ * compression. For small compressions (< 8 KB), that copy can dominate
+ * the cost of the compression.
+ *
+ * - The CDict's tables can be used in-place. In this model, compression is
+ * slower per input byte, because the compressor has to search two sets of
+ * tables. However, this model incurs no start-up cost (as long as the
+ * working context's tables can be reused). For small inputs, this can be
+ * faster than copying the CDict's tables.
+ *
+ * Zstd has a simple internal heuristic that selects which strategy to use
+ * at the beginning of a compression. However, if experimentation shows that
+ * Zstd is making poor choices, it is possible to override that choice with
+ * this enum.
+ */
+ ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
+ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
+ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
+} ZSTD_dictAttachPref_e;
+
+Frame size functions
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
`src` should point the start of a series of ZSTD encoded and/or skippable frames
@@ -418,22 +904,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
- srcSize must be >= ZSTD_frameHeaderSize_prefix.
+
srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
@return : size of the Frame Header,
or an error code (if srcSize is too small)
-Memory management
-
-size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
- These functions give the current memory usage of selected object.
- Object memory usage can evolve when re-used.
-
+Memory management
size_t ZSTD_estimateCCtxSize(int compressionLevel);
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -445,7 +921,7 @@ size_t ZSTD_estimateDCtxSize(void);
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1.
+ ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
Note : CCtx size estimation is only correct for single-threaded compression.
@@ -458,7 +934,7 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1.
+ ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
Note : CStream size estimation is only correct for single-threaded compression.
ZSTD_DStream memory budget depends on window Size.
This information can be passed manually, using ZSTD_estimateDStreamSize,
@@ -513,12 +989,13 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< t
-Advanced compression functions
+Advanced compression functions
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
Create a digested dictionary for compression
- Dictionary content is simply referenced, and therefore stays in dictBuffer.
- It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict
+ Dictionary content is just referenced, not duplicated.
+ As a consequence, `dictBuffer` **must** outlive CDict,
+ and its content must remain unmodified throughout the lifetime of CDict.
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
@@ -540,22 +1017,120 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< t
both values are optional, select `0` if unknown.
-size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- ZSTD_parameters params);
- Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter
+
size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
+ Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure)
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
- Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams);
+
Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
-Advanced decompression functions
+size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+ Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
+ It saves some memory, but also requires that `dict` outlives its usage within `cctx`
+
+
+size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+ Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
+ how to load the dictionary (by copy ? by reference ?)
+ and how to interpret it (automatic ? force raw mode ? full mode only ?)
+
+
+size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+ Same as ZSTD_CCtx_refPrefix(), but gives finer control over
+ how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?)
+
+
+size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+ Get the requested compression parameter value, selected by enum ZSTD_cParameter,
+ and store it into int* value.
+ @return : 0, or an error code (which can be tested with ZSTD_isError()).
+
+
+
+ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
+ Quick howto :
+ - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
+ - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
+ an existing ZSTD_CCtx_params structure.
+ This is similar to
+ ZSTD_CCtx_setParameter().
+ - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ an existing CCtx.
+ These parameters will be applied to
+ all subsequent frames.
+ - ZSTD_compressStream2() : Do compression using the CCtx.
+ - ZSTD_freeCCtxParams() : Free the memory.
+
+ This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ for static allocation of CCtx for single-threaded compression.
+
+
+
+size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+ Reset params to default values.
+
+
+
+size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+ Initializes the compression parameters of cctxParams according to
+ compression level. All other parameters are reset to their default values.
+
+
+
+size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+ Initializes the compression and frame parameters of cctxParams according to
+ params. All other parameters are reset to their default values.
+
+
+
+size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+ Similar to ZSTD_CCtx_setParameter.
+ Set one compression parameter, selected by enum ZSTD_cParameter.
+ Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+
+
+
+size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+ Similar to ZSTD_CCtx_getParameter.
+ Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+
+
+
+size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
+ Apply a set of ZSTD_CCtx_params to the compression context.
+ This can be done even after compression is started,
+ if nbWorkers==0, this will have no impact until a new compression is started.
+ if nbWorkers>=1, new parameters will be picked up at next job,
+ with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
+
+
+
+size_t ZSTD_compressStream2_simpleArgs (
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos,
+ ZSTD_EndDirective endOp);
+ Same as ZSTD_compressStream2(),
+ but using only integral types as arguments.
+ This variant might be helpful for binders from dynamic languages
+ which have troubles handling structures containing memory pointers.
+
+
+
+Advanced decompression functions
unsigned ZSTD_isFrame(const void* buffer, size_t size);
Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -595,7 +1170,56 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< t
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
-Advanced streaming functions
+size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ Same as ZSTD_DCtx_loadDictionary(),
+ but references `dict` content instead of copying it into `dctx`.
+ This saves memory if `dict` remains around.,
+ However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression.
+
+
+size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+ Same as ZSTD_DCtx_loadDictionary(),
+ but gives direct control over
+ how to load the dictionary (by copy ? by reference ?)
+ and how to interpret it (automatic ? force raw mode ? full mode only ?).
+
+
+size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+ Same as ZSTD_DCtx_refPrefix(), but gives finer control over
+ how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?)
+
+
+size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+ Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ This protects a decoder context from reserving too much memory for itself (potential attack scenario).
+ This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
+ @return : 0, or an error code (which can be tested using ZSTD_isError()).
+
+
+
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+ Instruct the decoder context about what kind of data to decode next.
+ This instruction is mandatory to decode data without a fully-formed header,
+ such ZSTD_f_zstd1_magicless for example.
+ @return : 0, or an error code (which can be tested using ZSTD_isError()).
+
+
+size_t ZSTD_decompressStream_simpleArgs (
+ ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos);
+ Same as ZSTD_decompressStream(),
+ but using only integral types as arguments.
+ This can be helpful for binders from dynamic languages
+ which have troubles handling structures containing memory pointers.
+
+
+
+Advanced streaming functions
Warning : most of these functions are now redundant with the Advanced API.
+ Once Advanced API reaches "stable" status,
+ redundant functions will be deprecated, and then at some point removed.
+
Advanced Streaming compression functions
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
@@ -605,7 +1229,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
- start a new compression job, using same parameters from previous job.
+
start a new frame, using same parameters from previous frame.
This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
Note that zcs must be init at least once before using ZSTD_resetCStream().
If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
@@ -635,25 +1259,23 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
+ there is no active job (could be checked with ZSTD_frameProgression()), or
+ oldest job is still actively compressing data,
but everything it has produced has also been flushed so far,
- therefore flushing speed is currently limited by production speed of oldest job
- irrespective of the speed of concurrent newer jobs.
+ therefore flush speed is limited by production speed of oldest job
+ irrespective of the speed of concurrent (and newer) jobs.
-Advanced Streaming decompression functions
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
-size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
+Advanced Streaming decompression functions
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */
size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
-Buffer-less and synchronous inner streaming functions
+Buffer-less and synchronous inner streaming functions
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience.
-Buffer-less streaming compression (synchronous mode)
+Buffer-less streaming compression (synchronous mode)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -689,7 +1311,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);
/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
-Buffer-less streaming decompression (synchronous mode)
+Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
@@ -770,513 +1392,24 @@ typedef struct {
unsigned dictID;
unsigned checksumFlag;
} ZSTD_frameHeader;
-/** ZSTD_getFrameHeader() :
- * decode Frame Header, or requires larger `srcSize`.
- * @return : 0, `zfhPtr` is correctly filled,
- * >0, `srcSize` is too small, value is wanted `srcSize` amount,
- * or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+
+ZSTD_getFrameHeader() :
decode Frame Header, or requires larger `srcSize`.
+ @return : 0, `zfhPtr` is correctly filled,
+ >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ or an error code, which can be tested using ZSTD_isError()
+
+
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+
+size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
-
-typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
-
-New advanced API (experimental)
-
-typedef enum {
- /* Opened question : should we have a format ZSTD_f_auto ?
- * Today, it would mean exactly the same as ZSTD_f_zstd1.
- * But, in the future, should several formats become supported,
- * on the compression side, it would mean "default format".
- * On the decompression side, it would mean "automatic format detection",
- * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
- * Since meaning is a little different, another option could be to define different enums for compression and decompression.
- * This question could be kept for later, when there are actually multiple formats to support,
- * but there is also the question of pinning enum values, and pinning value `0` is especially important */
- ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
- ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number.
- * Useful to save 4 bytes per generated frame.
- * Decoder cannot recognise automatically this format, requiring instructions. */
-} ZSTD_format_e;
-
-typedef enum {
- /* Note: this enum and the behavior it controls are effectively internal
- * implementation details of the compressor. They are expected to continue
- * to evolve and should be considered only in the context of extremely
- * advanced performance tuning.
- *
- * Zstd currently supports the use of a CDict in two ways:
- *
- * - The contents of the CDict can be copied into the working context. This
- * means that the compression can search both the dictionary and input
- * while operating on a single set of internal tables. This makes
- * the compression faster per-byte of input. However, the initial copy of
- * the CDict's tables incurs a fixed cost at the beginning of the
- * compression. For small compressions (< 8 KB), that copy can dominate
- * the cost of the compression.
- *
- * - The CDict's tables can be used in-place. In this model, compression is
- * slower per input byte, because the compressor has to search two sets of
- * tables. However, this model incurs no start-up cost (as long as the
- * working context's tables can be reused). For small inputs, this can be
- * faster than copying the CDict's tables.
- *
- * Zstd has a simple internal heuristic that selects which strategy to use
- * at the beginning of a compression. However, if experimentation shows that
- * Zstd is making poor choices, it is possible to override that choice with
- * this enum.
- */
- ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
- ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
- ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
-} ZSTD_dictAttachPref_e;
-
-typedef enum {
- /* compression format */
- ZSTD_p_format = 10, /* See ZSTD_format_e enum definition.
- * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
-
- /* compression parameters */
- ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
- * Default level is ZSTD_CLEVEL_DEFAULT==3.
- * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
- * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type.
- * Note 2 : setting a level sets all default values of other compression parameters.
- * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */
- ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
- * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
- * Special: value 0 means "use default windowLog".
- * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
- * requires explicitly allowing such window size during decompression stage. */
- ZSTD_p_hashLog, /* Size of the initial probe table, as a power of 2.
- * Resulting table size is (1 << (hashLog+2)).
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
- * Larger tables improve compression ratio of strategies <= dFast,
- * and improve speed of strategies > dFast.
- * Special: value 0 means "use default hashLog". */
- ZSTD_p_chainLog, /* Size of the multi-probe search table, as a power of 2.
- * Resulting table size is (1 << (chainLog+2)).
- * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
- * Larger tables result in better and slower compression.
- * This parameter is useless when using "fast" strategy.
- * Note it's still useful when using "dfast" strategy,
- * in which case it defines a secondary probe table.
- * Special: value 0 means "use default chainLog". */
- ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
- * More attempts result in better and slower compression.
- * This parameter is useless when using "fast" and "dFast" strategies.
- * Special: value 0 means "use default searchLog". */
- ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
- * Larger values make faster compression and decompression, but decrease ratio.
- * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
- * Note that currently, for all strategies < btopt, effective minimum is 4.
- * , for all strategies > fast, effective maximum is 6.
- * Special: value 0 means "use default minMatchLength". */
- ZSTD_p_targetLength, /* Impact of this field depends on strategy.
- * For strategies btopt & btultra:
- * Length of Match considered "good enough" to stop search.
- * Larger values make compression stronger, and slower.
- * For strategy fast:
- * Distance between match sampling.
- * Larger values make compression faster, and weaker.
- * Special: value 0 means "use default targetLength". */
- ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
- * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
- * The higher the value of selected strategy, the more complex it is,
- * resulting in stronger and slower compression.
- * Special: value 0 means "use default strategy". */
-
- ZSTD_p_enableLongDistanceMatching=160, /* Enable long distance matching.
- * This parameter is designed to improve compression ratio
- * for large inputs, by finding large matches at long distance.
- * It increases memory usage and window size.
- * Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB
- * except when expressly set to a different value. */
- ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2.
- * Larger values increase memory usage and compression ratio,
- * but decrease compression speed.
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
- * default: windowlog - 7.
- * Special: value 0 means "automatically determine hashlog". */
- ZSTD_p_ldmMinMatch, /* Minimum match size for long distance matcher.
- * Larger/too small values usually decrease compression ratio.
- * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
- * Special: value 0 means "use default value" (default: 64). */
- ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution.
- * Larger values improve collision resolution but decrease compression speed.
- * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX .
- * Special: value 0 means "use default value" (default: 3). */
- ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table.
- * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
- * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
- * Larger values improve compression speed.
- * Deviating far from default value will likely result in a compression ratio decrease.
- * Special: value 0 means "automatically determine hashEveryLog". */
-
- /* frame parameters */
- ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
- * Content size must be known at the beginning of compression,
- * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
- ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
- ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
-
- /* multi-threading parameters */
- /* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD).
- * They return an error otherwise. */
- ZSTD_p_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
- * When nbWorkers >= 1, triggers asynchronous mode :
- * ZSTD_compress_generic() consumes some input, flush some output if possible, and immediately gives back control to caller,
- * while compression work is performed in parallel, within worker threads.
- * (note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call).
- * More workers improve speed, but also increase memory usage.
- * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
- ZSTD_p_jobSize, /* Size of a compression job. This value is enforced only in non-blocking mode.
- * Each compression job is completed in parallel, so this value indirectly controls the nb of active threads.
- * 0 means default, which is dynamically determined based on compression parameters.
- * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
- * The minimum size is automatically and transparently enforced */
- ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
- * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-
- /* =================================================================== */
- /* experimental parameters - no stability guaranteed */
- /* =================================================================== */
-
- ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
- * even when referencing into Dictionary content (default:0) */
- ZSTD_p_forceAttachDict, /* Controls whether the contents of a CDict are
- * used in place, or whether they are copied into
- * the working context.
- *
- * Accepts values from the ZSTD_dictAttachPref_e
- * enum. See the comments on that enum for an
- * explanation of the feature.
- */
-} ZSTD_cParameter;
-
-size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
- Set one compression parameter, selected by enum ZSTD_cParameter.
- Setting a parameter is generally only possible during frame initialization (before starting compression).
- Exception : when using multi-threading mode (nbThreads >= 1),
- following parameters can be updated _during_ compression (within same frame):
- => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
- new parameters will be active on next job, or after a flush().
- Note : when `value` type is not unsigned (int, or enum), cast it to unsigned for proper type checking.
- @result : informational value (typically, value being set, correctly clamped),
- or an error code (which can be tested with ZSTD_isError()).
-
-
-size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value);
- Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-
-
-
-size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
- Total input data size to be compressed as a single frame.
- This value will be controlled at the end, and result in error if not respected.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Note 1 : 0 means zero, empty.
- In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
- Note 2 : If all data is provided and consumed in a single round,
- this value is overriden by srcSize instead.
-
-
-size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
- Create an internal CDict from `dict` buffer.
- Decompression will have to use same dictionary.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Special: Adding a NULL (or 0-size) dictionary invalidates previous dictionary,
- meaning "return to no-dictionary mode".
- Note 1 : Dictionary will be used for all future compression jobs.
- To return to "no-dictionary" situation, load a NULL dictionary
- Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
- For this reason, compression parameters cannot be changed anymore after loading a dictionary.
- It's also a CPU consuming operation, with non-negligible impact on latency.
- Note 3 :`dict` content will be copied internally.
- Use ZSTD_CCtx_loadDictionary_byReference() to reference dictionary content instead.
- In such a case, dictionary buffer must outlive its users.
- Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
- to precisely select how dictionary content must be interpreted.
-
-
-size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
- Reference a prepared dictionary, to be used for all next compression jobs.
- Note that compression parameters are enforced from within CDict,
- and supercede any compression parameter previously set within CCtx.
- The dictionary will remain valid for future compression jobs using same CCtx.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Special : adding a NULL CDict means "return to no-dictionary mode".
- Note 1 : Currently, only one dictionary can be managed.
- Adding a new dictionary effectively "discards" any previous one.
- Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
-
-
-size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
- const void* prefix, size_t prefixSize);
-size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
- const void* prefix, size_t prefixSize,
- ZSTD_dictContentType_e dictContentType);
- Reference a prefix (single-usage dictionary) for next compression job.
- Decompression will need same prefix to properly regenerate data.
- Compressing with a prefix is similar in outcome as performing a diff and compressing it,
- but performs much faster, especially during decompression (compression speed is tunable with compression level).
- Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
- Note 1 : Prefix buffer is referenced. It **must** outlive compression job.
- Its contain must remain unmodified up to end of compression (ZSTD_e_end).
- Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
- ensure that the window size is large enough to contain the entire source.
- See ZSTD_p_windowLog.
- Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- It's a CPU consuming operation, with non-negligible impact on latency.
- If there is a need to use same prefix multiple times, consider loadDictionary instead.
- Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
- Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
-
-
-void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
- Return a CCtx to clean state.
- Useful after an error, or to interrupt an ongoing compression job and start a new one.
- Any internal data not yet flushed is cancelled.
- The parameters and dictionary are kept unchanged, to reset them use ZSTD_CCtx_resetParameters().
-
-
-
-size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx);
- All parameters are back to default values (compression level is ZSTD_CLEVEL_DEFAULT).
- Dictionary (if any) is dropped.
- Resetting parameters is only possible during frame initialization (before starting compression).
- To reset the context use ZSTD_CCtx_reset().
- @return 0 or an error code (which can be checked with ZSTD_isError()).
-
-
-
-typedef enum {
- ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
- ZSTD_e_flush, /* flush any data provided so far,
- * it creates (at least) one new block, that can be decoded immediately on reception;
- * frame will continue: any future data can still reference previously compressed data, improving compression. */
- ZSTD_e_end /* flush any remaining data and close current frame.
- * any additional data starts a new frame.
- * each frame is independent (does not reference any content from previous frame). */
-} ZSTD_EndDirective;
-
-size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input,
- ZSTD_EndDirective endOp);
- Behave about the same as ZSTD_compressStream. To note :
- - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
- - Compression parameters cannot be changed once compression is started.
- - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
- - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
- and then immediately returns, just indicating that there is some data remaining to be flushed.
- The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
- - @return provides a minimum amount of data remaining to be flushed from internal buffers
- or an error code, which can be tested using ZSTD_isError().
- if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
- This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
- For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
- - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
- only ZSTD_e_end or ZSTD_e_flush operations are allowed.
- Before starting a new compression job, or changing compression parameters,
- it is required to fully flush internal buffers.
-
-
-
-size_t ZSTD_compress_generic_simpleArgs (
- ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos,
- ZSTD_EndDirective endOp);
- Same as ZSTD_compress_generic(),
- but using only integral types as arguments.
- Argument list is larger than ZSTD_{in,out}Buffer,
- but can be helpful for binders from dynamic languages
- which have troubles handling structures containing memory pointers.
-
-
-
-ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
-size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
- Quick howto :
- - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
- an existing ZSTD_CCtx_params structure.
- This is similar to
- ZSTD_CCtx_setParameter().
- - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
- an existing CCtx.
- These parameters will be applied to
- all subsequent compression jobs.
- - ZSTD_compress_generic() : Do compression using the CCtx.
- - ZSTD_freeCCtxParams() : Free the memory.
-
- This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
- for static allocation for single-threaded compression.
-
-
-
-size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
- Reset params to default values.
-
-
-
-size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
- Initializes the compression parameters of cctxParams according to
- compression level. All other parameters are reset to their default values.
-
-
-
-size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
- Initializes the compression and frame parameters of cctxParams according to
- params. All other parameters are reset to their default values.
-
-
-
-size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
- Similar to ZSTD_CCtx_setParameter.
- Set one compression parameter, selected by enum ZSTD_cParameter.
- Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
- Note : when `value` is an enum, cast it to unsigned for proper type checking.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-
-
-
-size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned* value);
- Similar to ZSTD_CCtx_getParameter.
- Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-
-
-
-size_t ZSTD_CCtx_setParametersUsingCCtxParams(
- ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
- Apply a set of ZSTD_CCtx_params to the compression context.
- This can be done even after compression is started,
- if nbWorkers==0, this will have no impact until a new compression is started.
- if nbWorkers>=1, new parameters will be picked up at next job,
- with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
-
-
-
-Advanced decompression API
/* ==================================== */
-
-size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
- Create an internal DDict from dict buffer,
- to be used to decompress next frames.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
- meaning "return to no-dictionary mode".
- Note 1 : `dict` content will be copied internally.
- Use ZSTD_DCtx_loadDictionary_byReference()
- to reference dictionary content instead.
- In which case, the dictionary buffer must outlive its users.
- Note 2 : Loading a dictionary involves building tables,
- which has a non-negligible impact on CPU usage and latency.
- Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
- how dictionary content will be interpreted and loaded.
-
-
-
-size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
- Reference a prepared dictionary, to be used to decompress next frames.
- The dictionary remains active for decompression of future frames using same DCtx.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Note 1 : Currently, only one dictionary can be managed.
- Referencing a new dictionary effectively "discards" any previous one.
- Special : adding a NULL DDict means "return to no-dictionary mode".
- Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
-
-
-
-size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
- const void* prefix, size_t prefixSize);
-size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
- const void* prefix, size_t prefixSize,
- ZSTD_dictContentType_e dictContentType);
- Reference a prefix (single-usage dictionary) for next compression job.
- This is the reverse operation of ZSTD_CCtx_refPrefix(),
- and must use the same prefix as the one used during compression.
- Prefix is **only used once**. Reference is discarded at end of frame.
- End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
- Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
- Note 2 : Prefix buffer is referenced. It **must** outlive decompression job.
- Prefix buffer must remain unmodified up to the end of frame,
- reached when ZSTD_DCtx_decompress_generic() returns 0.
- Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
- Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
- Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
- A fulldict prefix is more costly though.
-
-
-
-size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
- Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
- This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
- This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
- By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
- @return : 0, or an error code (which can be tested using ZSTD_isError()).
-
-
-
-size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
- Instruct the decoder context about what kind of data to decode next.
- This instruction is mandatory to decode data without a fully-formed header,
- such ZSTD_f_zstd1_magicless for example.
- @return : 0, or an error code (which can be tested using ZSTD_isError()).
-
-
-
-size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
- const void* src, size_t srcSize, ZSTD_format_e format);
same as ZSTD_getFrameHeader(),
with added capability to select a format (like ZSTD_f_zstd1_magicless)
-size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input);
- Behave the same as ZSTD_decompressStream.
- Decompression parameters cannot be changed once decompression is started.
- @return : an error code, which can be tested using ZSTD_isError()
- if >0, a hint, nb of expected input bytes for next invocation.
- `0` means : a frame has just been fully decoded and flushed.
-
-
-
-size_t ZSTD_decompress_generic_simpleArgs (
- ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos);
- Same as ZSTD_decompress_generic(),
- but using only integral types as arguments.
- Argument list is larger than ZSTD_{in,out}Buffer,
- but can be helpful for binders from dynamic languages
- which have troubles handling structures containing memory pointers.
-
-
-
-void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
- Return a DCtx to clean state.
- If a decompression was ongoing, any internal data not yet flushed is cancelled.
- All parameters are back to default values, including sticky ones.
- Dictionary (if any) is dropped.
- Parameters can be modified again after a reset.
-
-
-
-Block level API
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+
+Block level API
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -1290,10 +1423,10 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
+ copyCCtx() and copyDCtx() can be used too
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
- + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
+ + For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
- In which case, nothing is produced into `dst`.
+ In which case, nothing is produced into `dst` !
+ User must test for such outcome and deal directly with uncompressed data
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
+ In case of multiple successive blocks, should some of them be uncompressed,
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index e6286548..edeb74b9 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -78,7 +78,6 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
#define BIT0 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
-#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 99a00701..f05b5e10 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -61,7 +61,7 @@ static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager)
memset(cctx, 0, sizeof(*cctx));
cctx->customMem = memManager;
cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
- { size_t const err = ZSTD_CCtx_resetParameters(cctx);
+ { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
assert(!ZSTD_isError(err));
(void)err;
}
@@ -128,7 +128,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx)
#ifdef ZSTD_MULTITHREAD
return ZSTDMT_sizeof_CCtx(cctx->mtctx);
#else
- (void) cctx;
+ (void)cctx;
return 0;
#endif
}
@@ -226,6 +226,144 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
return ret;
}
+ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
+{
+ ZSTD_bounds bounds = { 0, 0, 0 };
+
+ switch(param)
+ {
+ case ZSTD_c_compressionLevel:
+ bounds.lowerBound = ZSTD_minCLevel();
+ bounds.upperBound = ZSTD_maxCLevel();
+ return bounds;
+
+ case ZSTD_c_windowLog:
+ bounds.lowerBound = ZSTD_WINDOWLOG_MIN;
+ bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_hashLog:
+ bounds.lowerBound = ZSTD_HASHLOG_MIN;
+ bounds.upperBound = ZSTD_HASHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_chainLog:
+ bounds.lowerBound = ZSTD_CHAINLOG_MIN;
+ bounds.upperBound = ZSTD_CHAINLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_searchLog:
+ bounds.lowerBound = ZSTD_SEARCHLOG_MIN;
+ bounds.upperBound = ZSTD_SEARCHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_minMatch:
+ bounds.lowerBound = ZSTD_MINMATCH_MIN;
+ bounds.upperBound = ZSTD_MINMATCH_MAX;
+ return bounds;
+
+ case ZSTD_c_targetLength:
+ bounds.lowerBound = ZSTD_TARGETLENGTH_MIN;
+ bounds.upperBound = ZSTD_TARGETLENGTH_MAX;
+ return bounds;
+
+ case ZSTD_c_compressionStrategy:
+ bounds.lowerBound = (int)ZSTD_fast;
+ bounds.upperBound = (int)ZSTD_btultra; /* note : how to ensure at compile time that this is the highest value strategy ? */
+ return bounds;
+
+ case ZSTD_c_contentSizeFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_checksumFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_dictIDFlag:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_nbWorkers:
+ bounds.lowerBound = 0;
+#ifdef ZSTD_MULTITHREAD
+ bounds.upperBound = ZSTDMT_NBWORKERS_MAX;
+#else
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_jobSize:
+ bounds.lowerBound = 0;
+#ifdef ZSTD_MULTITHREAD
+ bounds.upperBound = ZSTDMT_JOBSIZE_MAX;
+#else
+ bounds.upperBound = 0;
+#endif
+ return bounds;
+
+ case ZSTD_c_overlapSizeLog:
+ bounds.lowerBound = ZSTD_OVERLAPLOG_MIN;
+ bounds.upperBound = ZSTD_OVERLAPLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_enableLongDistanceMatching:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_ldmHashLog:
+ bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN;
+ bounds.upperBound = ZSTD_LDM_HASHLOG_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmMinMatch:
+ bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN;
+ bounds.upperBound = ZSTD_LDM_MINMATCH_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmBucketSizeLog:
+ bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN;
+ bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX;
+ return bounds;
+
+ case ZSTD_c_ldmHashRateLog:
+ bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN;
+ bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX;
+ return bounds;
+
+ /* experimental parameters */
+ case ZSTD_c_rsyncable:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_forceMaxWindow :
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ case ZSTD_c_format:
+ ZSTD_STATIC_ASSERT((int)ZSTD_f_zstd1 < (int)ZSTD_f_zstd1_magicless);
+ bounds.lowerBound = (int)ZSTD_f_zstd1;
+ bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
+ return bounds;
+
+ case ZSTD_c_forceAttachDict:
+ bounds.lowerBound = 0;
+ bounds.upperBound = 1;
+ return bounds;
+
+ default:
+ { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 };
+ return boundError;
+ }
+ }
+}
+
#define CLAMPCHECK(val,min,max) { \
if (((val)<(min)) | ((val)>(max))) { \
return ERROR(parameter_outOfBound); \
@@ -236,39 +374,39 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
{
switch(param)
{
- case ZSTD_p_compressionLevel:
- case ZSTD_p_hashLog:
- case ZSTD_p_chainLog:
- case ZSTD_p_searchLog:
- case ZSTD_p_minMatch:
- case ZSTD_p_targetLength:
- case ZSTD_p_compressionStrategy:
+ case ZSTD_c_compressionLevel:
+ case ZSTD_c_hashLog:
+ case ZSTD_c_chainLog:
+ case ZSTD_c_searchLog:
+ case ZSTD_c_minMatch:
+ case ZSTD_c_targetLength:
+ case ZSTD_c_compressionStrategy:
return 1;
- case ZSTD_p_format:
- case ZSTD_p_windowLog:
- case ZSTD_p_contentSizeFlag:
- case ZSTD_p_checksumFlag:
- case ZSTD_p_dictIDFlag:
- case ZSTD_p_forceMaxWindow :
- case ZSTD_p_nbWorkers:
- case ZSTD_p_jobSize:
- case ZSTD_p_overlapSizeLog:
- case ZSTD_p_rsyncable:
- case ZSTD_p_enableLongDistanceMatching:
- case ZSTD_p_ldmHashLog:
- case ZSTD_p_ldmMinMatch:
- case ZSTD_p_ldmBucketSizeLog:
- case ZSTD_p_ldmHashEveryLog:
- case ZSTD_p_forceAttachDict:
+ case ZSTD_c_format:
+ case ZSTD_c_windowLog:
+ case ZSTD_c_contentSizeFlag:
+ case ZSTD_c_checksumFlag:
+ case ZSTD_c_dictIDFlag:
+ case ZSTD_c_forceMaxWindow :
+ case ZSTD_c_nbWorkers:
+ case ZSTD_c_jobSize:
+ case ZSTD_c_overlapSizeLog:
+ case ZSTD_c_rsyncable:
+ case ZSTD_c_enableLongDistanceMatching:
+ case ZSTD_c_ldmHashLog:
+ case ZSTD_c_ldmMinMatch:
+ case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_ldmHashRateLog:
+ case ZSTD_c_forceAttachDict:
default:
return 0;
}
}
-size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
+size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
{
- DEBUGLOG(4, "ZSTD_CCtx_setParameter (%u, %u)", (U32)param, value);
+ DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value);
if (cctx->streamStage != zcss_init) {
if (ZSTD_isUpdateAuthorized(param)) {
cctx->cParamsChanged = 1;
@@ -278,52 +416,52 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
switch(param)
{
- case ZSTD_p_format :
+ case ZSTD_c_format :
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_compressionLevel:
+ case ZSTD_c_compressionLevel:
if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_windowLog:
- case ZSTD_p_hashLog:
- case ZSTD_p_chainLog:
- case ZSTD_p_searchLog:
- case ZSTD_p_minMatch:
- case ZSTD_p_targetLength:
- case ZSTD_p_compressionStrategy:
+ case ZSTD_c_windowLog:
+ case ZSTD_c_hashLog:
+ case ZSTD_c_chainLog:
+ case ZSTD_c_searchLog:
+ case ZSTD_c_minMatch:
+ case ZSTD_c_targetLength:
+ case ZSTD_c_compressionStrategy:
if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_contentSizeFlag:
- case ZSTD_p_checksumFlag:
- case ZSTD_p_dictIDFlag:
+ case ZSTD_c_contentSizeFlag:
+ case ZSTD_c_checksumFlag:
+ case ZSTD_c_dictIDFlag:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
+ case ZSTD_c_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content.
* default : 0 when using a CDict, 1 when using a Prefix */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_forceAttachDict:
+ case ZSTD_c_forceAttachDict:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_nbWorkers:
- if ((value>0) && cctx->staticSize) {
+ case ZSTD_c_nbWorkers:
+ if ((value!=0) && cctx->staticSize) {
return ERROR(parameter_unsupported); /* MT not compatible with static alloc */
}
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_jobSize:
- case ZSTD_p_overlapSizeLog:
- case ZSTD_p_rsyncable:
+ case ZSTD_c_jobSize:
+ case ZSTD_c_overlapSizeLog:
+ case ZSTD_c_rsyncable:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_enableLongDistanceMatching:
- case ZSTD_p_ldmHashLog:
- case ZSTD_p_ldmMinMatch:
- case ZSTD_p_ldmBucketSizeLog:
- case ZSTD_p_ldmHashEveryLog:
+ case ZSTD_c_enableLongDistanceMatching:
+ case ZSTD_c_ldmHashLog:
+ case ZSTD_c_ldmMinMatch:
+ case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_ldmHashRateLog:
if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@@ -331,20 +469,20 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
}
}
-size_t ZSTD_CCtxParam_setParameter(
- ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned value)
+size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
+ ZSTD_cParameter param, int value)
{
- DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%u, %u)", (U32)param, value);
+ DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%i, %i)", (int)param, value);
switch(param)
{
- case ZSTD_p_format :
- if (value > (unsigned)ZSTD_f_zstd1_magicless)
+ case ZSTD_c_format :
+ if (value > (int)ZSTD_f_zstd1_magicless)
return ERROR(parameter_unsupported);
CCtxParams->format = (ZSTD_format_e)value;
return (size_t)CCtxParams->format;
- case ZSTD_p_compressionLevel : {
- int cLevel = (int)value; /* cast expected to restore negative sign */
+ case ZSTD_c_compressionLevel : {
+ int cLevel = value;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevel) { /* 0 : does not change current level */
CCtxParams->compressionLevel = cLevel;
@@ -353,228 +491,228 @@ size_t ZSTD_CCtxParam_setParameter(
return 0; /* return type (size_t) cannot represent negative values */
}
- case ZSTD_p_windowLog :
- if (value>0) /* 0 => use default */
+ case ZSTD_c_windowLog :
+ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CCtxParams->cParams.windowLog = value;
return CCtxParams->cParams.windowLog;
- case ZSTD_p_hashLog :
- if (value>0) /* 0 => use default */
+ case ZSTD_c_hashLog :
+ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CCtxParams->cParams.hashLog = value;
return CCtxParams->cParams.hashLog;
- case ZSTD_p_chainLog :
- if (value>0) /* 0 => use default */
+ case ZSTD_c_chainLog :
+ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CCtxParams->cParams.chainLog = value;
return CCtxParams->cParams.chainLog;
- case ZSTD_p_searchLog :
- if (value>0) /* 0 => use default */
+ case ZSTD_c_searchLog :
+ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CCtxParams->cParams.searchLog = value;
return value;
- case ZSTD_p_minMatch :
- if (value>0) /* 0 => use default */
- CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
- CCtxParams->cParams.searchLength = value;
- return CCtxParams->cParams.searchLength;
+ case ZSTD_c_minMatch :
+ if (value!=0) /* 0 => use default */
+ CLAMPCHECK(value, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
+ CCtxParams->cParams.minMatch = value;
+ return CCtxParams->cParams.minMatch;
- case ZSTD_p_targetLength :
+ case ZSTD_c_targetLength :
/* all values are valid. 0 => use default */
CCtxParams->cParams.targetLength = value;
return CCtxParams->cParams.targetLength;
- case ZSTD_p_compressionStrategy :
- if (value>0) /* 0 => use default */
- CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
+ case ZSTD_c_compressionStrategy :
+ if (value!=0) /* 0 => use default */
+ CLAMPCHECK(value, (int)ZSTD_fast, (int)ZSTD_btultra);
CCtxParams->cParams.strategy = (ZSTD_strategy)value;
return (size_t)CCtxParams->cParams.strategy;
- case ZSTD_p_contentSizeFlag :
+ case ZSTD_c_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */
- DEBUGLOG(4, "set content size flag = %u", (value>0));
- CCtxParams->fParams.contentSizeFlag = value > 0;
+ DEBUGLOG(4, "set content size flag = %u", (value!=0));
+ CCtxParams->fParams.contentSizeFlag = value != 0;
return CCtxParams->fParams.contentSizeFlag;
- case ZSTD_p_checksumFlag :
+ case ZSTD_c_checksumFlag :
/* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
- CCtxParams->fParams.checksumFlag = value > 0;
+ CCtxParams->fParams.checksumFlag = value != 0;
return CCtxParams->fParams.checksumFlag;
- case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
- DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
+ case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
+ DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
CCtxParams->fParams.noDictIDFlag = !value;
return !CCtxParams->fParams.noDictIDFlag;
- case ZSTD_p_forceMaxWindow :
- CCtxParams->forceWindow = (value > 0);
+ case ZSTD_c_forceMaxWindow :
+ CCtxParams->forceWindow = (value != 0);
return CCtxParams->forceWindow;
- case ZSTD_p_forceAttachDict : {
+ case ZSTD_c_forceAttachDict : {
const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
CLAMPCHECK(pref, ZSTD_dictDefaultAttach, ZSTD_dictForceCopy);
CCtxParams->attachDictPref = pref;
return CCtxParams->attachDictPref;
}
- case ZSTD_p_nbWorkers :
+ case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
- if (value>0) return ERROR(parameter_unsupported);
+ if (value!=0) return ERROR(parameter_unsupported);
return 0;
#else
return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value);
#endif
- case ZSTD_p_jobSize :
+ case ZSTD_c_jobSize :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value);
#endif
- case ZSTD_p_overlapSizeLog :
+ case ZSTD_c_overlapSizeLog :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value);
#endif
- case ZSTD_p_rsyncable :
+ case ZSTD_c_rsyncable :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value);
#endif
- case ZSTD_p_enableLongDistanceMatching :
- CCtxParams->ldmParams.enableLdm = (value>0);
+ case ZSTD_c_enableLongDistanceMatching :
+ CCtxParams->ldmParams.enableLdm = (value!=0);
return CCtxParams->ldmParams.enableLdm;
- case ZSTD_p_ldmHashLog :
- if (value>0) /* 0 ==> auto */
+ case ZSTD_c_ldmHashLog :
+ if (value!=0) /* 0 ==> auto */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CCtxParams->ldmParams.hashLog = value;
return CCtxParams->ldmParams.hashLog;
- case ZSTD_p_ldmMinMatch :
- if (value>0) /* 0 ==> default */
+ case ZSTD_c_ldmMinMatch :
+ if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
CCtxParams->ldmParams.minMatchLength = value;
return CCtxParams->ldmParams.minMatchLength;
- case ZSTD_p_ldmBucketSizeLog :
- if (value > ZSTD_LDM_BUCKETSIZELOG_MAX)
- return ERROR(parameter_outOfBound);
+ case ZSTD_c_ldmBucketSizeLog :
+ if (value!=0) /* 0 ==> default */
+ CLAMPCHECK(value, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX);
CCtxParams->ldmParams.bucketSizeLog = value;
return CCtxParams->ldmParams.bucketSizeLog;
- case ZSTD_p_ldmHashEveryLog :
+ case ZSTD_c_ldmHashRateLog :
if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
return ERROR(parameter_outOfBound);
- CCtxParams->ldmParams.hashEveryLog = value;
- return CCtxParams->ldmParams.hashEveryLog;
+ CCtxParams->ldmParams.hashRateLog = value;
+ return CCtxParams->ldmParams.hashRateLog;
default: return ERROR(parameter_unsupported);
}
}
-size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value)
+size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value)
{
return ZSTD_CCtxParam_getParameter(&cctx->requestedParams, param, value);
}
size_t ZSTD_CCtxParam_getParameter(
- ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned* value)
+ ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value)
{
switch(param)
{
- case ZSTD_p_format :
+ case ZSTD_c_format :
*value = CCtxParams->format;
break;
- case ZSTD_p_compressionLevel :
+ case ZSTD_c_compressionLevel :
*value = CCtxParams->compressionLevel;
break;
- case ZSTD_p_windowLog :
+ case ZSTD_c_windowLog :
*value = CCtxParams->cParams.windowLog;
break;
- case ZSTD_p_hashLog :
+ case ZSTD_c_hashLog :
*value = CCtxParams->cParams.hashLog;
break;
- case ZSTD_p_chainLog :
+ case ZSTD_c_chainLog :
*value = CCtxParams->cParams.chainLog;
break;
- case ZSTD_p_searchLog :
+ case ZSTD_c_searchLog :
*value = CCtxParams->cParams.searchLog;
break;
- case ZSTD_p_minMatch :
- *value = CCtxParams->cParams.searchLength;
+ case ZSTD_c_minMatch :
+ *value = CCtxParams->cParams.minMatch;
break;
- case ZSTD_p_targetLength :
+ case ZSTD_c_targetLength :
*value = CCtxParams->cParams.targetLength;
break;
- case ZSTD_p_compressionStrategy :
+ case ZSTD_c_compressionStrategy :
*value = (unsigned)CCtxParams->cParams.strategy;
break;
- case ZSTD_p_contentSizeFlag :
+ case ZSTD_c_contentSizeFlag :
*value = CCtxParams->fParams.contentSizeFlag;
break;
- case ZSTD_p_checksumFlag :
+ case ZSTD_c_checksumFlag :
*value = CCtxParams->fParams.checksumFlag;
break;
- case ZSTD_p_dictIDFlag :
+ case ZSTD_c_dictIDFlag :
*value = !CCtxParams->fParams.noDictIDFlag;
break;
- case ZSTD_p_forceMaxWindow :
+ case ZSTD_c_forceMaxWindow :
*value = CCtxParams->forceWindow;
break;
- case ZSTD_p_forceAttachDict :
+ case ZSTD_c_forceAttachDict :
*value = CCtxParams->attachDictPref;
break;
- case ZSTD_p_nbWorkers :
+ case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
assert(CCtxParams->nbWorkers == 0);
#endif
*value = CCtxParams->nbWorkers;
break;
- case ZSTD_p_jobSize :
+ case ZSTD_c_jobSize :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
*value = CCtxParams->jobSize;
break;
#endif
- case ZSTD_p_overlapSizeLog :
+ case ZSTD_c_overlapSizeLog :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
*value = CCtxParams->overlapSizeLog;
break;
#endif
- case ZSTD_p_rsyncable :
+ case ZSTD_c_rsyncable :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
*value = CCtxParams->rsyncable;
break;
#endif
- case ZSTD_p_enableLongDistanceMatching :
+ case ZSTD_c_enableLongDistanceMatching :
*value = CCtxParams->ldmParams.enableLdm;
break;
- case ZSTD_p_ldmHashLog :
+ case ZSTD_c_ldmHashLog :
*value = CCtxParams->ldmParams.hashLog;
break;
- case ZSTD_p_ldmMinMatch :
+ case ZSTD_c_ldmMinMatch :
*value = CCtxParams->ldmParams.minMatchLength;
break;
- case ZSTD_p_ldmBucketSizeLog :
+ case ZSTD_c_ldmBucketSizeLog :
*value = CCtxParams->ldmParams.bucketSizeLog;
break;
- case ZSTD_p_ldmHashEveryLog :
- *value = CCtxParams->ldmParams.hashEveryLog;
+ case ZSTD_c_ldmHashRateLog :
+ *value = CCtxParams->ldmParams.hashRateLog;
break;
default: return ERROR(parameter_unsupported);
}
@@ -672,18 +810,22 @@ size_t ZSTD_CCtx_refPrefix_advanced(
/*! ZSTD_CCtx_reset() :
* Also dumps dictionary */
-void ZSTD_CCtx_reset(ZSTD_CCtx* cctx)
+size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
{
- cctx->streamStage = zcss_init;
- cctx->pledgedSrcSizePlusOne = 0;
+ if ( (reset == ZSTD_reset_session_only)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ cctx->streamStage = zcss_init;
+ cctx->pledgedSrcSizePlusOne = 0;
+ }
+ if ( (reset == ZSTD_reset_parameters)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+ cctx->cdict = NULL;
+ return ZSTD_CCtxParams_reset(&cctx->requestedParams);
+ }
+ return 0;
}
-size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx)
-{
- if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
- cctx->cdict = NULL;
- return ZSTD_CCtxParams_reset(&cctx->requestedParams);
-}
/** ZSTD_checkCParams() :
control CParam values remain within authorized range.
@@ -694,7 +836,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
- CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+ CLAMPCHECK(cParams.minMatch, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0);
if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
return ERROR(parameter_outOfBound);
@@ -717,7 +859,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams)
CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
- CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+ CLAMP(cParams.minMatch, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0);
if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
cParams.targetLength = ZSTD_TARGETLENGTH_MAX;
@@ -791,7 +933,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog;
if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog;
if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog;
- if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength;
+ if (CCtxParams->cParams.minMatch) cParams.minMatch = CCtxParams->cParams.minMatch;
if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength;
if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy;
assert(!ZSTD_checkCParams(cParams));
@@ -804,7 +946,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
{
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
size_t const hSize = ((size_t)1) << cParams->hashLog;
- U32 const hashLog3 = (forCCtx && cParams->searchLength==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
size_t const hSize = ((size_t)1) << cParams->hashLog;
- U32 const hashLog3 = (forCCtx && cParams->searchLength==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
@@ -1175,13 +1317,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
/* Adjust long distance matching parameters */
ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
- assert(params.ldmParams.hashEveryLog < 32);
+ assert(params.ldmParams.hashRateLog < 32);
zc->ldmState.hashPower = ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
}
{ size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
- U32 const divider = (params.cParams.searchLength==3) ? 3 : 4;
+ U32 const divider = (params.cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq;
size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
@@ -2461,7 +2603,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
- ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength);
+ ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch);
cSize = 0;
goto out; /* don't even attempt compression below a certain srcSize */
}
@@ -2640,7 +2782,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
size_t pos=0;
assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
- if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
+ if (dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX) return ERROR(dstSize_tooSmall);
DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
!params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
@@ -3646,8 +3788,15 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
/*====== Compression ======*/
-MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
- const void* src, size_t srcSize)
+static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
+{
+ size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
+ if (hintInSize==0) hintInSize = cctx->blockSize;
+ return hintInSize;
+}
+
+static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
if (length) memcpy(dst, src, length);
@@ -3655,7 +3804,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
}
/** ZSTD_compressStream_generic():
- * internal function for all *compressStream*() variants and *compress_generic()
+ * internal function for all *compressStream*() variants
* non-static, because can be called from zstdmt_compress.c
* @return : hint size for next input */
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
@@ -3699,7 +3848,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ip = iend;
op += cSize;
zcs->frameEnded = 1;
- ZSTD_CCtx_reset(zcs);
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
someMoreWork = 0; break;
}
/* complete loading into inBuffer */
@@ -3752,7 +3901,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
if (zcs->frameEnded) {
DEBUGLOG(5, "Frame completed directly in outBuffer");
someMoreWork = 0;
- ZSTD_CCtx_reset(zcs);
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
}
break;
}
@@ -3780,7 +3929,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
if (zcs->frameEnded) {
DEBUGLOG(5, "Frame completed on flush");
someMoreWork = 0;
- ZSTD_CCtx_reset(zcs);
+ ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
break;
}
zcs->streamStage = zcss_load;
@@ -3795,28 +3944,34 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
input->pos = ip - istart;
output->pos = op - ostart;
if (zcs->frameEnded) return 0;
- { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
- if (hintInSize==0) hintInSize = zcs->blockSize;
- return hintInSize;
+ return ZSTD_nextInputSizeHint(zcs);
+}
+
+static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx)
+{
+#ifdef ZSTD_MULTITHREAD
+ if (cctx->appliedParams.nbWorkers >= 1) {
+ assert(cctx->mtctx != NULL);
+ return ZSTDMT_nextInputSizeHint(cctx->mtctx);
}
+#endif
+ return ZSTD_nextInputSizeHint(cctx);
+
}
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
- /* check conditions */
- if (output->pos > output->size) return ERROR(GENERIC);
- if (input->pos > input->size) return ERROR(GENERIC);
-
- return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue);
+ CHECK_F( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) );
+ return ZSTD_nextInputSizeHint_MTorST(zcs);
}
-size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input,
- ZSTD_EndDirective endOp)
+size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp)
{
- DEBUGLOG(5, "ZSTD_compress_generic, endOp=%u ", (U32)endOp);
+ DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (U32)endOp);
/* check conditions */
if (output->pos > output->size) return ERROR(GENERIC);
if (input->pos > input->size) return ERROR(GENERIC);
@@ -3828,7 +3983,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
- DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage");
+ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */
params.cParams = ZSTD_getCParamsFromCCtxParams(
&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
@@ -3841,7 +3996,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (params.nbWorkers > 0) {
/* mt context creation */
if (cctx->mtctx == NULL) {
- DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbWorkers=%u",
+ DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
params.nbWorkers);
cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem);
if (cctx->mtctx == NULL) return ERROR(memory_allocation);
@@ -3863,6 +4018,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
assert(cctx->streamStage == zcss_load);
assert(cctx->appliedParams.nbWorkers == 0);
} }
+ /* end of transparent initialization stage */
/* compression stage */
#ifdef ZSTD_MULTITHREAD
@@ -3874,18 +4030,18 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
{ size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
- ZSTD_CCtx_reset(cctx);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
}
- DEBUGLOG(5, "completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic");
+ DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
return flushMin;
} }
#endif
CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
- DEBUGLOG(5, "completed ZSTD_compress_generic");
+ DEBUGLOG(5, "completed ZSTD_compressStream2");
return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
}
-size_t ZSTD_compress_generic_simpleArgs (
+size_t ZSTD_compressStream2_simpleArgs (
ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos,
@@ -3893,13 +4049,33 @@ size_t ZSTD_compress_generic_simpleArgs (
{
ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
ZSTD_inBuffer input = { src, srcSize, *srcPos };
- /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
- size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp);
+ /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
+ size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
*dstPos = output.pos;
*srcPos = input.pos;
return cErr;
}
+size_t ZSTD_compress2(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+ { size_t oPos = 0;
+ size_t iPos = 0;
+ size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
+ dst, dstCapacity, &oPos,
+ src, srcSize, &iPos,
+ ZSTD_e_end);
+ if (ZSTD_isError(result)) return result;
+ if (result != 0) { /* compression not completed, due to lack of output space */
+ assert(oPos == dstCapacity);
+ return ERROR(dstSize_tooSmall);
+ }
+ assert(iPos == srcSize); /* all input is expected consumed */
+ return oPos;
+ }
+}
/*====== Finalize ======*/
@@ -3908,20 +4084,20 @@ size_t ZSTD_compress_generic_simpleArgs (
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
ZSTD_inBuffer input = { NULL, 0, 0 };
- if (output->pos > output->size) return ERROR(GENERIC);
- CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) );
- return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
+ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
}
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
ZSTD_inBuffer input = { NULL, 0, 0 };
- if (output->pos > output->size) return ERROR(GENERIC);
- CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) );
+ size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
+ CHECK_F( remainingToFlush );
+ if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */
+ /* single thread mode : attempt to calculate remaining to flush more precisely */
{ size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4;
- size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize;
+ size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize;
DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (U32)toFlush);
return toFlush;
}
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 608efd5a..680bd914 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -161,7 +161,7 @@ typedef struct {
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
- U32 hashEveryLog; /* Log number of entries to skip */
+ U32 hashRateLog; /* Log number of entries to skip */
U32 windowLog; /* Window log for the LDM */
} ldmParams_t;
diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c
index 7b9e18e7..47faf6d6 100644
--- a/lib/compress/zstd_double_fast.c
+++ b/lib/compress/zstd_double_fast.c
@@ -18,7 +18,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashLarge = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
- U32 const mls = cParams->searchLength;
+ U32 const mls = cParams->minMatch;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const base = ms->window.base;
@@ -309,7 +309,7 @@ size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
- const U32 mls = ms->cParams.searchLength;
+ const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
@@ -329,7 +329,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
- const U32 mls = ms->cParams.searchLength;
+ const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
@@ -483,7 +483,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
- U32 const mls = ms->cParams.searchLength;
+ U32 const mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c
index 24774651..d54a5342 100644
--- a/lib/compress/zstd_fast.c
+++ b/lib/compress/zstd_fast.c
@@ -18,7 +18,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hBits = cParams->hashLog;
- U32 const mls = cParams->searchLength;
+ U32 const mls = cParams->minMatch;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
@@ -235,7 +235,7 @@ size_t ZSTD_compressBlock_fast(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
- U32 const mls = cParams->searchLength;
+ U32 const mls = cParams->minMatch;
assert(ms->dictMatchState == NULL);
switch(mls)
{
@@ -256,7 +256,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
- U32 const mls = cParams->searchLength;
+ U32 const mls = cParams->minMatch;
assert(ms->dictMatchState != NULL);
switch(mls)
{
@@ -375,7 +375,7 @@ size_t ZSTD_compressBlock_fast_extDict(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
- U32 const mls = cParams->searchLength;
+ U32 const mls = cParams->minMatch;
switch(mls)
{
default: /* includes case 3 */
diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
index e86f225a..53f998a4 100644
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@@ -392,7 +392,7 @@ ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
- switch(ms->cParams.searchLength)
+ switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
@@ -408,7 +408,7 @@ static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
- switch(ms->cParams.searchLength)
+ switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
@@ -424,7 +424,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
- switch(ms->cParams.searchLength)
+ switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
@@ -469,7 +469,7 @@ static U32 ZSTD_insertAndFindFirstIndex_internal(
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
- return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.searchLength);
+ return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
}
@@ -566,7 +566,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
- switch(ms->cParams.searchLength)
+ switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
@@ -582,7 +582,7 @@ static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
- switch(ms->cParams.searchLength)
+ switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
@@ -598,7 +598,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
- switch(ms->cParams.searchLength)
+ switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 3f180ddb..e9800680 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -37,8 +37,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
}
- if (params->hashEveryLog == 0) {
- params->hashEveryLog = params->windowLog < params->hashLog
+ if (params->hashRateLog == 0) {
+ params->hashRateLog = params->windowLog < params->hashLog
? 0
: params->windowLog - params->hashLog;
}
@@ -119,20 +119,20 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
*
* Gets the small hash, checksum, and tag from the rollingHash.
*
- * If the tag matches (1 << ldmParams.hashEveryLog)-1, then
+ * If the tag matches (1 << ldmParams.hashRateLog)-1, then
* creates an ldmEntry from the offset, and inserts it into the hash table.
*
* hBits is the length of the small hash, which is the most significant hBits
* of rollingHash. The checksum is the next 32 most significant bits, followed
- * by ldmParams.hashEveryLog bits that make up the tag. */
+ * by ldmParams.hashRateLog bits that make up the tag. */
static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
U64 const rollingHash,
U32 const hBits,
U32 const offset,
ldmParams_t const ldmParams)
{
- U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog);
- U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
+ U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
+ U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
if (tag == tagMask) {
U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
@@ -247,8 +247,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
U64 const hashPower = ldmState->hashPower;
U32 const hBits = params->hashLog - params->bucketSizeLog;
U32 const ldmBucketSize = 1U << params->bucketSizeLog;
- U32 const hashEveryLog = params->hashEveryLog;
- U32 const ldmTagMask = (1U << params->hashEveryLog) - 1;
+ U32 const hashRateLog = params->hashRateLog;
+ U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
/* Prefix and extDict parameters */
U32 const dictLimit = ldmState->window.dictLimit;
U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@@ -283,7 +283,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
lastHashed = ip;
/* Do not insert and do not look for a match */
- if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) {
+ if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
ip++;
continue;
}
@@ -543,7 +543,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
void const* src, size_t srcSize)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
- unsigned const minMatch = cParams->searchLength;
+ unsigned const minMatch = cParams->minMatch;
ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
/* Input bounds */
diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h
index 5310e174..a4784612 100644
--- a/lib/compress/zstd_ldm.h
+++ b/lib/compress/zstd_ldm.h
@@ -21,7 +21,7 @@ extern "C" {
* Long distance matching
***************************************/
-#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
/**
* ZSTD_ldm_generateSequences():
@@ -87,7 +87,7 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params);
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
/** ZSTD_ldm_adjustParameters() :
- * If the params->hashEveryLog is not set, set it to its default value based on
+ * If the params->hashRateLog is not set, set it to its default value based on
* windowLog and params->hashLog.
*
* Ensures that params->bucketSizeLog is <= params->hashLog (setting it to
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 6dfc85de..d5a59f6b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -488,7 +488,7 @@ void ZSTD_updateTree_internal(
}
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
- ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict);
+ ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
}
FORCE_INLINE_TEMPLATE
@@ -728,7 +728,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
ZSTD_match_t* matches, U32 const lengthToBeat)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
- U32 const matchLengthSearch = cParams->searchLength;
+ U32 const matchLengthSearch = cParams->minMatch;
DEBUGLOG(8, "ZSTD_BtGetAllMatches");
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
@@ -796,7 +796,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
- U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
+ U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
ZSTD_match_t* const matches = optStatePtr->matchTable;
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 43afbc1b..4ce3dda0 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -10,8 +10,6 @@
/* ====== Tuning parameters ====== */
-#define ZSTDMT_NBWORKERS_MAX 200
-#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */
#define ZSTDMT_OVERLAPLOG_DEFAULT 6
@@ -469,7 +467,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
- assert(params.ldmParams.hashEveryLog < 32);
+ assert(params.ldmParams.hashRateLog < 32);
serialState->ldmState.hashPower =
ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
} else {
@@ -674,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} else { /* srcStart points at reloaded section */
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
- { size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob);
+ { size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
}
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
@@ -1846,7 +1844,9 @@ typedef struct {
* Otherwise, we will load as many bytes as possible and instruct the caller
* to continue as normal.
*/
-static syncPoint_t findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) {
+static syncPoint_t
+findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
+{
BYTE const* const istart = (BYTE const*)input.src + input.pos;
U64 const primePower = mtctx->rsync.primePower;
U64 const hitMask = mtctx->rsync.hitMask;
@@ -1910,6 +1910,13 @@ static syncPoint_t findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuf
return syncPoint;
}
+size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx)
+{
+ size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled;
+ if (hintInSize==0) hintInSize = mtctx->targetSectionSize;
+ return hintInSize;
+}
+
/** ZSTDMT_compressStream_generic() :
* internal use only - exposed to be invoked from zstd_compress.c
* assumption : output and input are valid (pos <= size)
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index b6bcb9e5..c60ec832 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -28,6 +28,14 @@
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
+/* === Constants === */
+#define ZSTDMT_NBWORKERS_MAX 200
+#ifndef ZSTDMT_JOBSIZE_MIN
+# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
+#endif
+#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1 GB)) /* note : limited by `jobSize` type, which is `int` */
+
+
/* === Memory management === */
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
@@ -52,6 +60,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
+ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
@@ -60,10 +69,6 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
/* === Advanced functions and parameters === */
-#ifndef ZSTDMT_JOBSIZE_MIN
-# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
-#endif
-
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 549cc758..d0b6063d 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -37,12 +37,12 @@
* It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
*/
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
+# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
#endif
/*!
* NO_FORWARD_PROGRESS_MAX :
- * maximum allowed nb of calls to ZSTD_decompressStream() and ZSTD_decompress_generic()
+ * maximum allowed nb of calls to ZSTD_decompressStream()
* without any forward progress
* (defined as: no byte read from input, and no byte flushed to output)
* before triggering an error.
@@ -89,8 +89,8 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
static size_t ZSTD_startingInputLength(ZSTD_format_e format)
{
size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
- ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE :
- ZSTD_frameHeaderSize_prefix;
+ ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE :
+ ZSTD_FRAMEHEADERSIZE_PREFIX;
ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
@@ -187,7 +187,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
if (size < ZSTD_FRAMEIDSIZE) return 0;
{ U32 const magic = MEM_readLE32(buffer);
if (magic == ZSTD_MAGICNUMBER) return 1;
- if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
}
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(buffer, size)) return 1;
@@ -242,10 +242,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
if ( (format != ZSTD_f_zstd1_magicless)
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
- if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
/* skippable frame */
- if (srcSize < ZSTD_skippableHeaderSize)
- return ZSTD_skippableHeaderSize; /* magic number + frame length */
+ if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
+ return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
memset(zfhPtr, 0, sizeof(*zfhPtr));
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
zfhPtr->frameType = ZSTD_skippableFrame;
@@ -352,15 +352,15 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{
unsigned long long totalDstSize = 0;
- while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+ while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
U32 const magicNumber = MEM_readLE32(src);
- if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize;
- if (srcSize < ZSTD_skippableHeaderSize)
+ if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ERROR(srcSize_wrong);
skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE)
- + ZSTD_skippableHeaderSize;
+ + ZSTD_SKIPPABLEHEADERSIZE;
if (srcSize < skippableSize) {
return ZSTD_CONTENTSIZE_ERROR;
}
@@ -434,9 +434,9 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
if (ZSTD_isLegacy(src, srcSize))
return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
#endif
- if ( (srcSize >= ZSTD_skippableHeaderSize)
- && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
- return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE);
+ if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
+ && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START ) {
+ return ZSTD_SKIPPABLEHEADERSIZE + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE);
} else {
const BYTE* ip = (const BYTE*)src;
const BYTE* const ipstart = ip;
@@ -547,11 +547,11 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
/* check */
- if (remainingSrcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize)
+ if (remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize)
return ERROR(srcSize_wrong);
/* Frame Header */
- { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
+ { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX);
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
if (remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize)
return ERROR(srcSize_wrong);
@@ -632,7 +632,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
dictSize = ZSTD_DDict_dictSize(ddict);
}
- while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+ while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(src, srcSize)) {
@@ -659,12 +659,12 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
{ U32 const magicNumber = MEM_readLE32(src);
DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
(U32)magicNumber, (U32)ZSTD_MAGICNUMBER);
- if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize;
- if (srcSize < ZSTD_skippableHeaderSize)
+ if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ERROR(srcSize_wrong);
skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE)
- + ZSTD_skippableHeaderSize;
+ + ZSTD_SKIPPABLEHEADERSIZE;
if (srcSize < skippableSize) return ERROR(srcSize_wrong);
src = (const BYTE *)src + skippableSize;
@@ -790,9 +790,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
assert(src != NULL);
if (dctx->format == ZSTD_f_zstd1) { /* allows header */
assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
- if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
memcpy(dctx->headerBuffer, src, srcSize);
- dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */
+ dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
dctx->stage = ZSTDds_decodeSkippableHeader;
return 0;
} }
@@ -897,8 +897,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
case ZSTDds_decodeSkippableHeader:
assert(src != NULL);
- assert(srcSize <= ZSTD_skippableHeaderSize);
- memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */
+ assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
+ memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
dctx->stage = ZSTDds_skipFrame;
return 0;
@@ -1192,7 +1192,7 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
/* ZSTD_initDStream_usingDict() :
- * return : expected size, aka ZSTD_frameHeaderSize_prefix.
+ * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
* this function cannot fail */
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{
@@ -1200,7 +1200,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
zds->streamStage = zdss_init;
zds->noForwardProgress = 0;
CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
- return ZSTD_frameHeaderSize_prefix;
+ return ZSTD_FRAMEHEADERSIZE_PREFIX;
}
/* note : this variant can't fail */
@@ -1221,7 +1221,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
}
/* ZSTD_resetDStream() :
- * return : expected size, aka ZSTD_frameHeaderSize_prefix.
+ * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
* this function cannot fail */
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
{
@@ -1230,23 +1230,9 @@ size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0;
dctx->legacyVersion = 0;
dctx->hostageByte = 0;
- return ZSTD_frameHeaderSize_prefix;
+ return ZSTD_FRAMEHEADERSIZE_PREFIX;
}
-size_t ZSTD_setDStreamParameter(ZSTD_DStream* dctx,
- ZSTD_DStreamParameter_e paramType, unsigned paramValue)
-{
- if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
- switch(paramType)
- {
- default : return ERROR(parameter_unsupported);
- case DStream_p_maxWindowSize :
- DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10);
- dctx->maxWindowSize = paramValue ? paramValue : (U32)(-1);
- break;
- }
- return 0;
-}
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
@@ -1255,18 +1241,92 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
return 0;
}
+/* ZSTD_DCtx_setMaxWindowSize() :
+ * note : no direct equivalence in ZSTD_DCtx_setParameter,
+ * since this version sets windowSize, and the other sets windowLog */
size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
{
+ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
+ size_t const min = (size_t)1 << bounds.lowerBound;
+ size_t const max = (size_t)1 << bounds.upperBound;
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
+ if (maxWindowSize < min) return ERROR(parameter_outOfBound);
+ if (maxWindowSize > max) return ERROR(parameter_outOfBound);
dctx->maxWindowSize = maxWindowSize;
return 0;
}
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
{
- DEBUGLOG(4, "ZSTD_DCtx_setFormat : %u", (unsigned)format);
+ return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format);
+}
+
+ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
+{
+ ZSTD_bounds bounds = { 0, 0, 0 };
+ switch(dParam) {
+ case ZSTD_d_windowLogMax:
+ bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ bounds.upperBound = ZSTD_WINDOWLOG_MAX;
+ return bounds;
+ case ZSTD_d_format:
+ bounds.lowerBound = (int)ZSTD_f_zstd1;
+ bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
+ ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
+ return bounds;
+ default:;
+ }
+ bounds.error = ERROR(parameter_unsupported);
+ return bounds;
+}
+
+/* ZSTD_dParam_withinBounds:
+ * @return 1 if value is within dParam bounds,
+ * 0 otherwise */
+static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
+{
+ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
+ if (ZSTD_isError(bounds.error)) return 0;
+ if (value < bounds.lowerBound) return 0;
+ if (value > bounds.upperBound) return 0;
+ return 1;
+}
+
+#define CHECK_DBOUNDS(p,v) { \
+ if (!ZSTD_dParam_withinBounds(p, v)) \
+ return ERROR(parameter_outOfBound); \
+}
+
+size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
+{
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
- dctx->format = format;
+ switch(dParam) {
+ case ZSTD_d_windowLogMax:
+ CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
+ dctx->maxWindowSize = ((size_t)1) << value;
+ return 0;
+ case ZSTD_d_format:
+ CHECK_DBOUNDS(ZSTD_d_format, value);
+ dctx->format = (ZSTD_format_e)value;
+ return 0;
+ default:;
+ }
+ return ERROR(parameter_unsupported);
+}
+
+size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
+{
+ if ( (reset == ZSTD_reset_session_only)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ (void)ZSTD_initDStream(dctx);
+ }
+ if ( (reset == ZSTD_reset_parameters)
+ || (reset == ZSTD_reset_session_and_parameters) ) {
+ if (dctx->streamStage != zdss_init)
+ return ERROR(stage_wrong);
+ dctx->format = ZSTD_f_zstd1;
+ dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+ }
return 0;
}
@@ -1296,7 +1356,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
{
- U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable */
+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
ZSTD_frameHeader zfh;
size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(err)) return err;
@@ -1391,7 +1451,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->lhSize += remainingInput;
}
input->pos = input->size;
- return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
+ return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
assert(ip != NULL);
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
@@ -1419,7 +1479,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
DEBUGLOG(4, "Consume header");
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
- if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
zds->stage = ZSTDds_skipFrame;
} else {
@@ -1579,13 +1639,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
}
}
-
-size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
-{
- return ZSTD_decompressStream(dctx, output, input);
-}
-
-size_t ZSTD_decompress_generic_simpleArgs (
+size_t ZSTD_decompressStream_simpleArgs (
ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos)
@@ -1593,15 +1647,8 @@ size_t ZSTD_decompress_generic_simpleArgs (
ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
ZSTD_inBuffer input = { src, srcSize, *srcPos };
/* ZSTD_compress_generic() will check validity of dstPos and srcPos */
- size_t const cErr = ZSTD_decompress_generic(dctx, &output, &input);
+ size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
*dstPos = output.pos;
*srcPos = input.pos;
return cErr;
}
-
-void ZSTD_DCtx_reset(ZSTD_DCtx* dctx)
-{
- (void)ZSTD_initDStream(dctx);
- dctx->format = ZSTD_f_zstd1;
- dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
-}
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 2964b69f..c3c365c8 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -255,7 +255,7 @@ static dictItem ZDICT_analyzePos(
}
{ int i;
- U32 searchLength;
+ U32 mml;
U32 refinedStart = start;
U32 refinedEnd = end;
@@ -263,7 +263,7 @@ static dictItem ZDICT_analyzePos(
DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos);
DISPLAYLEVEL(4, "\n");
- for (searchLength = MINMATCHLENGTH ; ; searchLength++) {
+ for (mml = MINMATCHLENGTH ; ; mml++) {
BYTE currentChar = 0;
U32 currentCount = 0;
U32 currentID = refinedStart;
@@ -271,13 +271,13 @@ static dictItem ZDICT_analyzePos(
U32 selectedCount = 0;
U32 selectedID = currentID;
for (id =refinedStart; id < refinedEnd; id++) {
- if (b[suffix[id] + searchLength] != currentChar) {
+ if (b[suffix[id] + mml] != currentChar) {
if (currentCount > selectedCount) {
selectedCount = currentCount;
selectedID = currentID;
}
currentID = id;
- currentChar = b[ suffix[id] + searchLength];
+ currentChar = b[ suffix[id] + mml];
currentCount = 0;
}
currentCount ++;
diff --git a/lib/dll/libzstd.def b/lib/dll/libzstd.def
index 51d0c192..668c4b1c 100644
--- a/lib/dll/libzstd.def
+++ b/lib/dll/libzstd.def
@@ -78,7 +78,6 @@ EXPORTS
ZSTD_nextSrcSizeToDecompress
ZSTD_resetCStream
ZSTD_resetDStream
- ZSTD_setDStreamParameter
ZSTD_sizeof_CCtx
ZSTD_sizeof_CDict
ZSTD_sizeof_CStream
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index e852bb91..d7522025 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -240,17 +240,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
/* *************************************
* Types
***************************************/
-#define ZSTD_WINDOWLOG_MAX 26
-#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
-#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
-#define ZSTD_CONTENTLOG_MIN 4
-#define ZSTD_HASHLOG_MAX 28
-#define ZSTD_HASHLOG_MIN 4
-#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
-#define ZSTD_SEARCHLOG_MIN 1
-#define ZSTD_SEARCHLENGTH_MAX 7
-#define ZSTD_SEARCHLENGTH_MIN 4
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
diff --git a/lib/zstd.h b/lib/zstd.h
index 6eb2dd83..3e977aa0 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -74,13 +74,13 @@ extern "C" {
#define ZSTD_VERSION_RELEASE 8
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
+ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
-ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0+ */
+ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
/***************************************
* Default constant
@@ -110,7 +110,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
-/*! ZSTD_getFrameContentSize() : added in v1.3.0
+/*! ZSTD_getFrameContentSize() : requires v1.3.0+
* `src` should point to the start of a ZSTD encoded frame.
* `srcSize` must be at least as large as the frame header.
* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
@@ -167,8 +167,10 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
/*! ZSTD_compressCCtx() :
- * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
-ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx
+ * The function will compress at requested compression level,
+ * ignoring any other parameter */
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
@@ -184,8 +186,11 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
/*! ZSTD_decompressDCtx() :
- * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
-ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
+ * Same as ZSTD_decompress(),
+ * requires an allocated ZSTD_DCtx.
+ * Compatible with sticky parameters.
+ */
+ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
@@ -194,9 +199,12 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
* Simple dictionary API
***************************/
/*! ZSTD_compress_usingDict() :
- * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+ * Compression at an explicit compression level using a Dictionary.
+ * A dictionary can be any arbitrary data segment (also called a prefix),
+ * or a buffer with specified information (see dictBuilder/zdict.h).
* Note : This function loads the dictionary, resulting in significant startup delay.
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ * It's intended for a dictionary used only once.
+ * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -204,9 +212,10 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
int compressionLevel);
/*! ZSTD_decompress_usingDict() :
- * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+ * Decompression using a known Dictionary.
* Dictionary must be identical to the one used during compression.
* Note : This function loads the dictionary, resulting in significant startup delay.
+ * It's intended for a dictionary used only once.
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
@@ -214,17 +223,18 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* dict,size_t dictSize);
-/**********************************
+/***********************************
* Bulk processing dictionary API
- *********************************/
+ **********************************/
typedef struct ZSTD_CDict_s ZSTD_CDict;
/*! ZSTD_createCDict() :
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
+ * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict
- * Note : A ZSTD_CDict can be created with an empty dictionary, but it is inefficient for small data. */
+ * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
+ * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
@@ -234,11 +244,9 @@ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
/*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary.
- * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
- * Note that compression level is decided during dictionary creation.
- * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no)
- * Note : ZSTD_compress_usingCDict() can be used with a ZSTD_CDict created from an empty dictionary.
- * But it is inefficient for small data, and it is recommended to use ZSTD_compressCCtx(). */
+ * Recommended when same dictionary is used multiple times.
+ * Note : compression level is _decided at dictionary creation time_,
+ * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -249,7 +257,7 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
- * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
/*! ZSTD_freeDDict() :
@@ -258,7 +266,7 @@ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
/*! ZSTD_decompress_usingDDict() :
* Decompression using a digested Dictionary.
- * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
+ * Recommended when same dictionary is used multiple times. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -289,13 +297,17 @@ typedef struct ZSTD_outBuffer_s {
* A ZSTD_CStream object is required to track streaming operation.
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
-* since it will play nicer with system's memory, by re-using already allocated memory.
-* Use one separate ZSTD_CStream per thread for parallel execution.
+* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
*
-* Start a new compression by initializing ZSTD_CStream context.
-* Use ZSTD_initCStream() to start a new compression operation.
-* Use variants ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for streaming with dictionary (experimental section)
+* For parallel execution, use one separate ZSTD_CStream per thread.
+*
+* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
+*
+* Parameters are sticky : when starting a new compression on the same context,
+* it will re-use the same sticky parameters as previous compression session.
+* When in doubt, it's recommended to fully initialize the context before usage.
+* Use ZSTD_initCStream() to set the parameter to a selected compression level.
+* Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
*
* Use ZSTD_compressStream() as many times as necessary to consume input stream.
* The function will automatically update both `pos` fields within `input` and `output`.
@@ -304,12 +316,11 @@ typedef struct ZSTD_outBuffer_s {
* in which case `input.pos < input.size`.
* The caller must check if input has been entirely consumed.
* If not, the caller must make some room to receive more compressed data,
-* typically by emptying output buffer, or allocating a new output buffer,
* and then present again remaining input data.
-* @return : a size hint, preferred nb of bytes to use as input for next function call
-* or an error code, which can be tested using ZSTD_isError().
-* Note 1 : it's just a hint, to help latency a little, any other value will work fine.
-* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
+* @return : a size hint, preferred nb of bytes to use as input for next function call
+* or an error code, which can be tested using ZSTD_isError().
+* Note 1 : it's just a hint, to help latency a little, any value will work fine.
+* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
*
* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
* using ZSTD_flushStream(). `output->pos` will be updated.
@@ -353,25 +364,24 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
* ZSTD_DStream objects can be re-used multiple times.
*
-* Use ZSTD_initDStream() to start a new decompression operation,
-* or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
-* @return : recommended first input size
+* Use ZSTD_initDStream() to start a new decompression operation.
+* @return : recommended first input size
+* Alternatively, use advanced API to set specific properties.
*
* Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update both `pos` fields.
* If `input.pos < input.size`, some input has not been consumed.
* It's up to the caller to present again remaining data.
-* The function tries to flush all data decoded immediately, repecting buffer sizes.
+* The function tries to flush all data decoded immediately, respecting output buffer size.
* If `output.pos < output.size`, decoder has flushed everything it could.
-* But if `output.pos == output.size`, there is no such guarantee,
-* it's likely that some decoded data was not flushed and still remains within internal buffers.
+* But if `output.pos == output.size`, there might be some data left within internal buffers.,
* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
-* When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
* @return : 0 when a frame is completely decoded and fully flushed,
* or an error code, which can be tested using ZSTD_isError(),
* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
-* the return value is a suggested next input size (a hint for better latency)
-* that will never load more than the current frame.
+* the return value is a suggested next input size (just a hint for better latency)
+* that will never request more than the remaining frame size.
* *******************************************************************************/
typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
@@ -392,77 +402,590 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
-#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
-#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
-
/****************************************************************************************
* ADVANCED AND EXPERIMENTAL FUNCTIONS
****************************************************************************************
- * The definitions in this section are considered experimental.
- * They should never be used with a dynamic library, as prototypes may change in the future.
+ * The definitions in the following section are considered experimental.
* They are provided for advanced scenarios.
+ * They should never be used with a dynamic library, as prototypes may change in the future.
* Use them only in association with static linking.
* ***************************************************************************************/
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
+
+/****************************************************************************************
+ * Candidate API for promotion to stable status
+ ****************************************************************************************
+ * The following symbols and constants form the "staging area" :
+ * they are considered to join "stable API" by v1.4.0.
+ * The proposal is written so that it can be made stable "as is",
+ * though it's still possible to suggest improvements.
+ * Staging is in fact last chance for changes,
+ * the API is locked once reaching "stable" status.
+ * ***************************************************************************************/
+
+
+/* === Constants === */
+
+/* all magic numbers are supposed read/written to/from files/memory using little-endian convention */
+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
+#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
+#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
+#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
+
+#define ZSTD_BLOCKSIZELOG_MAX 17
+#define ZSTD_BLOCKSIZE_MAX (1<= first frame size
+ * @return : the compressed size of the first frame starting at `src`,
+ * suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ * or an error code if input is invalid */
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */
+
+/* === Memory management === */
+
+/*! ZSTD_sizeof_*() :
+ * These functions give the _current_ memory usage of selected object.
+ * Note that object memory usage can evolve (increase or decrease) over time. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+
+/***************************************
+* Advanced compression API
+***************************************/
+
+/* API design :
+ * Parameters are pushed one by one into an existing context,
+ * using ZSTD_CCtx_set*() functions.
+ * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
+ * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
+ * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
+ *
+ * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
+ *
+ * This API supercedes all other "advanced" API entry points in the experimental section.
+ * In the future, we expect to remove from experimental API entry points which are redundant with this API.
+ */
+
+
+/* Compression strategies, listed from fastest to strongest */
+typedef enum { ZSTD_fast=1,
+ ZSTD_dfast=2,
+ ZSTD_greedy=3,
+ ZSTD_lazy=4,
+ ZSTD_lazy2=5,
+ ZSTD_btlazy2=6,
+ ZSTD_btopt=7,
+ ZSTD_btultra=8
+ /* note : new strategies might be added in the future.
+ Only the order (from fast to strong) is guaranteed, not the exact position.
+ new strategy names might be introduced, pushing the maximum number upward */
+} ZSTD_strategy;
+
+
+typedef enum {
+
+ /* compression parameters */
+ ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
+ * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+ * Note 1 : it's possible to pass a negative compression level.
+ * Note 2 : setting a level sets all default values of other compression parameters */
+ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+ * Special: value 0 means "use default windowLog".
+ * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+ * requires explicitly allowing such window size at decompression stage if using streaming. */
+ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
+ * Resulting memory usage is (1 << (hashLog+2)).
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+ * Larger tables improve compression ratio of strategies <= dFast,
+ * and improve speed of strategies > dFast.
+ * Special: value 0 means "use default hashLog". */
+ ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
+ * Resulting memory usage is (1 << (chainLog+2)).
+ * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+ * Larger tables result in better and slower compression.
+ * This parameter is useless when using "fast" strategy.
+ * It's still useful when using "dfast" strategy,
+ * in which case it defines a secondary probe table.
+ * Special: value 0 means "use default chainLog". */
+ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
+ * More attempts result in better and slower compression.
+ * This parameter is useless when using "fast" and "dFast" strategies.
+ * Special: value 0 means "use default searchLog". */
+ ZSTD_c_minMatch=105, /* Minimum size of searched matches.
+ * Note that Zstandard can still find matches of smaller size,
+ * it just tweaks its search algorithm to look for this size and larger.
+ * Larger values increase compression and decompression speed, but decrease ratio.
+ * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
+ * , for all strategies > fast, effective maximum is 6.
+ * Special: value 0 means "use default minMatchLength". */
+ ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
+ * For strategies btopt & btultra:
+ * Length of Match considered "good enough" to stop search.
+ * Larger values make compression stronger, and slower.
+ * For strategy fast:
+ * Distance between match sampling.
+ * Larger values make compression faster, and weaker.
+ * Special: value 0 means "use default targetLength". */
+ ZSTD_c_compressionStrategy=107, /* See ZSTD_strategy enum definition.
+ * The higher the value of selected strategy, the more complex it is,
+ * resulting in stronger and slower compression.
+ * Special: value 0 means "use default strategy". */
+
+ /* LDM mode parameters */
+ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
+ * This parameter is designed to improve compression ratio
+ * for large inputs, by finding large matches at long distance.
+ * It increases memory usage and window size.
+ * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+ * except when expressly set to a different value. */
+ ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
+ * Larger values increase memory usage and compression ratio,
+ * but decrease compression speed.
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+ * default: windowlog - 7.
+ * Special: value 0 means "automatically determine hashlog". */
+ ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
+ * Larger/too small values usually decrease compression ratio.
+ * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+ * Special: value 0 means "use default value" (default: 64). */
+ ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
+ * Larger values improve collision resolution but decrease compression speed.
+ * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+ * Special: value 0 means "use default value" (default: 3). */
+ ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
+ * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+ * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+ * Larger values improve compression speed.
+ * Deviating far from default value will likely result in a compression ratio decrease.
+ * Special: value 0 means "automatically determine hashRateLog". */
+
+ /* frame parameters */
+ ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+ * Content size must be known at the beginning of compression,
+ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
+ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
+ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
+
+ /* multi-threading parameters */
+ /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+ * They return an error otherwise. */
+ ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
+ * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
+ * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+ * while compression work is performed in parallel, within worker threads.
+ * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+ * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+ * More workers improve speed, but also increase memory usage.
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
+ ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
+ * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+ * 0 means default, which is dynamically determined based on compression parameters.
+ * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
+ * The minimum size is automatically and transparently enforced */
+ ZSTD_c_overlapSizeLog=402, /* Size of previous job reloaded at the beginning of each job, as a fraction of window size.
+ * This value is enforced only when nbWorkers >= 1.
+ * Larger values increase compression ratio, but decrease speed.
+ * Values range from 0 (no overlap) to 9 (overlap a full windowSize).
+ * Each rank (except 0) increase/decrease load size by a factor 2
+ * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:w/256;
+ * default value is 6 : use 1/8th of windowSize */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_c_rsyncable
+ * ZSTD_c_format
+ * ZSTD_c_forceMaxWindow
+ * ZSTD_c_forceAttachDict
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly;
+ * also, the enums values themselves are unstable and can still change.
+ */
+ ZSTD_c_experimentalParam1=500,
+ ZSTD_c_experimentalParam2=10,
+ ZSTD_c_experimentalParam3=1000,
+ ZSTD_c_experimentalParam4=1001
+} ZSTD_cParameter;
+
+
+typedef struct {
+ size_t error;
+ int lowerBound;
+ int upperBound;
+} ZSTD_bounds;
+
+/*! ZSTD_cParam_getBounds() :
+ * All parameters must belong to an interval with lower and upper bounds,
+ * otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ * - an error status field, which must be tested using ZSTD_isError()
+ * - lower and upper bounds, both inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
+
+/*! ZSTD_CCtx_setParameter() :
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ * Setting a parameter is generally only possible during frame initialization (before starting compression).
+ * Exception : when using multi-threading mode (nbWorkers >= 1),
+ * the following parameters can be updated _during_ compression (within same frame):
+ * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
+ * new parameters will be active for next job only (after a flush()).
+ * @return : an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ * Total input data size to be compressed as a single frame.
+ * This value will be controlled at end of frame, and trigger an error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
+ * Note 2 : pledgedSrcSize is only valid once, for the next frame.
+ * It's discarded at the end of the frame.
+ * Note 3 : If all data is provided and consumed in a single round,
+ * this value is automatically overriden by srcSize instead.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+/*! ZSTD_CCtx_loadDictionary() :
+ * Create an internal CDict from `dict` buffer.
+ * Decompression will have to use same dictionary.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
+ * meaning "return to no-dictionary mode".
+ * Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
+ * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ * Note 2 : Loading a dictionary involves building tables.
+ * It's also a CPU consuming operation, with non-negligible impact on latency.
+ * Tables are dependent on compression parameters, and for this reason,
+ * compression parameters can no longer be changed after loading a dictionary.
+ * Note 3 :`dict` content will be copied internally.
+ * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
+ * In such a case, dictionary buffer must outlive its users.
+ * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
+ * to precisely select how dictionary content must be interpreted. */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() :
+ * Reference a prepared dictionary, to be used for all next compressed frames.
+ * Note that compression parameters are enforced from within CDict,
+ * and supercede any compression parameter previously set within CCtx.
+ * The dictionary will remain valid for future compressed frames using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special : Referencing a NULL CDict means "return to no-dictionary mode".
+ * Note 1 : Currently, only one dictionary can be managed.
+ * Referencing a new dictionary effectively "discards" any previous one.
+ * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() :
+ * Reference a prefix (single-usage dictionary) for next compressed frame.
+ * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
+ * Decompression will need same prefix to properly regenerate data.
+ * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
+ * but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
+ * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
+ * Its content must remain unmodified during compression.
+ * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
+ * ensure that the window size is large enough to contain the entire source.
+ * See ZSTD_c_windowLog.
+ * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ * It's a CPU consuming operation, with non-negligible impact on latency.
+ * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+ * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
+ const void* prefix, size_t prefixSize);
+
+
+typedef enum {
+ ZSTD_reset_session_only = 1,
+ ZSTD_reset_parameters = 2,
+ ZSTD_reset_session_and_parameters = 3
+} ZSTD_ResetDirective;
+
+/*! ZSTD_CCtx_reset() :
+ * There are 2 different things that can be reset, independently or jointly :
+ * - The session : will stop compressing current frame, and make CCtx ready to start a new one.
+ * Useful after an error, or to interrupt any ongoing compression.
+ * Any internal data not yet flushed is cancelled.
+ * Compression parameters and dictionary remain unchanged.
+ * They will be used to compress next frame.
+ * Resetting session never fails.
+ * - The parameters : changes all parameters back to "default".
+ * This removes any reference to any dictionary too.
+ * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
+ * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
+ * - Both : similar to resetting the session, followed by resetting parameters.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
+
+
+
+/*! ZSTD_compress2() :
+ * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ * ZSTD_compress2() always starts a new frame.
+ * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ * - The function is always blocking, returns when compression is completed.
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ * @return : compressed size written into `dst` (<= `dstCapacity),
+ * or an error code if it fails (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize);
+
+typedef enum {
+ ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+ ZSTD_e_flush=1, /* flush any data provided so far,
+ * it creates (at least) one new block, that can be decoded immediately on reception;
+ * frame will continue: any future data can still reference previously compressed data, improving compression. */
+ ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
+ * note that frame is only closed after compressed data is fully flushed (return value == 0).
+ * After that point, any additional data starts a new frame.
+ * note : each frame is independent (does not reference any content from previous frame). */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream2() :
+ * Behaves about the same as ZSTD_compressStream, with additional control on end directive.
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
+ * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
+ * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
+ * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+ * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
+ * and then immediately returns, just indicating that there is some data remaining to be flushed.
+ * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
+ * - @return provides a minimum amount of data remaining to be flushed from internal buffers
+ * or an error code, which can be tested using ZSTD_isError().
+ * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
+ * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
+ * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
+ * Before starting a new compression job, or changing compression parameters,
+ * it is required to fully flush internal buffers.
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
+ ZSTD_outBuffer* output,
+ ZSTD_inBuffer* input,
+ ZSTD_EndDirective endOp);
+
+
+
+/* ============================== */
+/* Advanced decompression API */
+/* ============================== */
+
+/* The advanced API pushes parameters one by one into an existing DCtx context.
+ * Parameters are sticky, and remain valid for all following frames
+ * using the same DCtx context.
+ * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
+ * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
+ * Therefore, no new decompression function is necessary.
+ */
+
+
+typedef enum {
+
+ ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
+ * the streaming API will refuse to allocate memory buffer
+ * in order to protect the host from unreasonable memory requirements.
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_c_format
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly
+ */
+ ZSTD_d_experimentalParam1=1000
+
+} ZSTD_dParameter;
+
+
+/*! ZSTD_dParam_getBounds() :
+ * All parameters must belong to an interval with lower and upper bounds,
+ * otherwise they will either trigger an error or be automatically clamped.
+ * @return : a structure, ZSTD_bounds, which contains
+ * - an error status field, which must be tested using ZSTD_isError()
+ * - both lower and upper bounds, inclusive
+ */
+ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
+
+/*! ZSTD_DCtx_setParameter() :
+ * Set one compression parameter, selected by enum ZSTD_dParameter.
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ * Setting a parameter is only possible during frame initialization (before starting decompression).
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
+
+
+/*! ZSTD_DCtx_loadDictionary() :
+ * Create an internal DDict from dict buffer,
+ * to be used to decompress next frames.
+ * The dictionary remains valid for all future frames, until explicitly invalidated.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
+ * meaning "return to no-dictionary mode".
+ * Note 1 : Loading a dictionary involves building tables,
+ * which has a non-negligible impact on CPU usage and latency.
+ * It's recommended to "load once, use many times", to amortize the cost
+ * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
+ * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
+ * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
+ * how dictionary content is loaded and interpreted.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_refDDict() :
+ * Reference a prepared dictionary, to be used to decompress next frames.
+ * The dictionary remains active for decompression of future frames using same DCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : Currently, only one dictionary can be managed.
+ * Referencing a new dictionary effectively "discards" any previous one.
+ * Special: referencing a NULL DDict means "return to no-dictionary mode".
+ * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
+
+/*! ZSTD_DCtx_refPrefix() :
+ * Reference a prefix (single-usage dictionary) to decompress next frame.
+ * This is the reverse operation of ZSTD_CCtx_refPrefix(),
+ * and must use the same prefix as the one used during compression.
+ * Prefix is **only used once**. Reference is discarded at end of frame.
+ * End of frame is reached when ZSTD_decompressStream() returns 0.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
+ * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
+ * Prefix buffer must remain unmodified up to the end of frame,
+ * reached when ZSTD_decompressStream() returns 0.
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
+ * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
+ * A full dictionary is more costly, as it requires building tables.
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
+ const void* prefix, size_t prefixSize);
+
+/*! ZSTD_DCtx_reset() :
+ * Return a DCtx to clean state.
+ * Session and parameters can be reset jointly or separately.
+ * Parameters can only be reset when no active frame is being decompressed.
+ * @return : 0, or an error code, which can be tested with ZSTD_isError()
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
+
+
+
+/****************************************************************************************
+ * experimental API (static linking only)
+ ****************************************************************************************
+ * The following symbols and constants
+ * are not planned to join "stable API" status in the near future.
+ * They can still change in future versions.
+ * Some of them are planned to remain in the static_only section indefinitely.
+ * Some of them might be removed in the future (especially when redundant with existing stable functions)
+ * ***************************************************************************************/
+
+#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
#define ZSTD_FRAMEHEADERSIZE_MIN 6
-#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
-static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX;
-static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
-static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
-static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
+#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
+#define ZSTD_SKIPPABLEHEADERSIZE 8
+/* compression parameter bounds */
+#define ZSTD_WINDOWLOG_MAX_32 30
+#define ZSTD_WINDOWLOG_MAX_64 31
+#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MIN 10
+#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
+#define ZSTD_HASHLOG_MIN 6
+#define ZSTD_CHAINLOG_MAX_32 29
+#define ZSTD_CHAINLOG_MAX_64 30
+#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
+#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
+#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
+#define ZSTD_SEARCHLOG_MIN 1
+#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
+#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
+#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
+
+#define ZSTD_OVERLAPLOG_MIN 0
+#define ZSTD_OVERLAPLOG_MAX 9
+
+#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
+ * requiring larger than (1<= first frame size
- * @return : the compressed size of the first frame starting at `src`,
- * suitable to pass to `ZSTD_decompress` or similar,
- * or an error code if input is invalid */
-ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-
/*! ZSTD_findDecompressedSize() :
* `src` should point the start of a series of ZSTD encoded and/or skippable frames
* `srcSize` must be the _exact_ size of this series
@@ -521,7 +1081,7 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_frameHeaderSize() :
- * srcSize must be >= ZSTD_frameHeaderSize_prefix.
+ * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* @return : size of the Frame Header,
* or an error code (if srcSize is too small) */
ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
@@ -531,16 +1091,6 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
* Memory management
***************************************/
-/*! ZSTD_sizeof_*() :
- * These functions give the current memory usage of selected object.
- * Object memory usage can evolve when re-used. */
-ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-
/*! ZSTD_estimate*() :
* These functions make it possible to estimate memory usage
* of a future {D,C}Ctx, before its creation.
@@ -548,7 +1098,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1.
+ * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
* Note : CCtx size estimation is only correct for single-threaded compression. */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -560,7 +1110,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1.
+ * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
* Note : CStream size estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size.
* This information can be passed manually, using ZSTD_estimateDStreamSize,
@@ -623,6 +1173,7 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType);
+
/*! Custom memory allocation :
* These prototypes make it possible to pass your own allocation/free functions.
* ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
@@ -657,8 +1208,9 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
/*! ZSTD_createCDict_byReference() :
* Create a digested dictionary for compression
- * Dictionary content is simply referenced, and therefore stays in dictBuffer.
- * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
+ * Dictionary content is just referenced, not duplicated.
+ * As a consequence, `dictBuffer` **must** outlive CDict,
+ * and its content must remain unmodified throughout the lifetime of CDict. */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_getCParams() :
@@ -681,22 +1233,161 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() :
-* Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */
-ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- ZSTD_parameters params);
+ * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
+ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
/*! ZSTD_compress_usingCDict_advanced() :
-* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
+ * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict,
+ ZSTD_frameParameters fParams);
-/*--- Advanced decompression functions ---*/
+/*! ZSTD_CCtx_loadDictionary_byReference() :
+ * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
+ * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_loadDictionary_advanced() :
+ * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
+ * how to load the dictionary (by copy ? by reference ?)
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_CCtx_refPrefix_advanced() :
+ * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/* === experimental parameters === */
+/* these parameters can be used with ZSTD_setParameter()
+ * they are not guaranteed to remain supported in the future */
+
+ /* Enables rsyncable mode,
+ * which makes compressed files more rsync friendly
+ * by adding periodic synchronization points to the compressed data.
+ * The target average block size is ZSTD_c_jobSize / 2.
+ * It's possible to modify the job size to increase or decrease
+ * the granularity of the synchronization point.
+ * Once the jobSize is smaller than the window size,
+ * it will result in compression ratio degradation.
+ * NOTE 1: rsyncable mode only works when multithreading is enabled.
+ * NOTE 2: rsyncable performs poorly in combination with long range mode,
+ * since it will decrease the effectiveness of synchronization points,
+ * though mileage may vary.
+ * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
+ * If the selected compression level is already running significantly slower,
+ * the overall speed won't be significantly impacted.
+ */
+ #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
+
+/* Select a compression format.
+ * The value must be of type ZSTD_format_e.
+ * See ZSTD_format_e enum definition for details */
+#define ZSTD_c_format ZSTD_c_experimentalParam2
+
+/* Force back-reference distances to remain < windowSize,
+ * even when referencing into Dictionary content (default:0) */
+#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
+
+/* Controls whether the contents of a CDict
+ * are used in place, or copied into the working context.
+ * Accepts values from the ZSTD_dictAttachPref_e enum.
+ * See the comments on that enum for an explanation of the feature. */
+#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
+
+/*! ZSTD_CCtx_getParameter() :
+ * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
+ * and store it into int* value.
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+
+
+/*! ZSTD_CCtx_params :
+ * Quick howto :
+ * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
+ * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
+ * an existing ZSTD_CCtx_params structure.
+ * This is similar to
+ * ZSTD_CCtx_setParameter().
+ * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
+ * an existing CCtx.
+ * These parameters will be applied to
+ * all subsequent frames.
+ * - ZSTD_compressStream2() : Do compression using the CCtx.
+ * - ZSTD_freeCCtxParams() : Free the memory.
+ *
+ * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
+ * for static allocation of CCtx for single-threaded compression.
+ */
+ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
+
+/*! ZSTD_CCtxParams_reset() :
+ * Reset params to default values.
+ */
+ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+
+/*! ZSTD_CCtxParams_init() :
+ * Initializes the compression parameters of cctxParams according to
+ * compression level. All other parameters are reset to their default values.
+ */
+ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+
+/*! ZSTD_CCtxParams_init_advanced() :
+ * Initializes the compression and frame parameters of cctxParams according to
+ * params. All other parameters are reset to their default values.
+ */
+ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+
+/*! ZSTD_CCtxParam_setParameter() :
+ * Similar to ZSTD_CCtx_setParameter.
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+
+/*! ZSTD_CCtxParam_getParameter() :
+ * Similar to ZSTD_CCtx_getParameter.
+ * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+
+/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
+ * Apply a set of ZSTD_CCtx_params to the compression context.
+ * This can be done even after compression is started,
+ * if nbWorkers==0, this will have no impact until a new compression is started.
+ * if nbWorkers>=1, new parameters will be picked up at next job,
+ * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
+
+/*! ZSTD_compressStream2_simpleArgs() :
+ * Same as ZSTD_compressStream2(),
+ * but using only integral types as arguments.
+ * This variant might be helpful for binders from dynamic languages
+ * which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs (
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos,
+ ZSTD_EndDirective endOp);
+
+
+/***************************************
+* Advanced decompression functions
+***************************************/
/*! ZSTD_isFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -737,9 +1428,64 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+/*! ZSTD_DCtx_loadDictionary_byReference() :
+ * Same as ZSTD_DCtx_loadDictionary(),
+ * but references `dict` content instead of copying it into `dctx`.
+ * This saves memory if `dict` remains around.,
+ * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_DCtx_loadDictionary_advanced() :
+ * Same as ZSTD_DCtx_loadDictionary(),
+ * but gives direct control over
+ * how to load the dictionary (by copy ? by reference ?)
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
+ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_refPrefix_advanced() :
+ * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
+ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+
+/*! ZSTD_DCtx_setMaxWindowSize() :
+ * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
+ * This protects a decoder context from reserving too much memory for itself (potential attack scenario).
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ */
+ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+
+/* ZSTD_d_format
+ * experimental parameter,
+ * allowing selection between ZSTD_format_e input compression formats
+ */
+#define ZSTD_d_format ZSTD_d_experimentalParam1
+
+/*! ZSTD_DCtx_setFormat() :
+ * Instruct the decoder context about what kind of data to decode next.
+ * This instruction is mandatory to decode data without a fully-formed header,
+ * such ZSTD_f_zstd1_magicless for example.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+
+/*! ZSTD_decompressStream_simpleArgs() :
+ * Same as ZSTD_decompressStream(),
+ * but using only integral types as arguments.
+ * This can be helpful for binders from dynamic languages
+ * which have troubles handling structures containing memory pointers.
+ */
+ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
+ ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity, size_t* dstPos,
+ const void* src, size_t srcSize, size_t* srcPos);
+
/********************************************************************
* Advanced streaming functions
+* Warning : most of these functions are now redundant with the Advanced API.
+* Once Advanced API reaches "stable" status,
+* redundant functions will be deprecated, and then at some point removed.
********************************************************************/
/*===== Advanced Streaming compression functions =====*/
@@ -751,7 +1497,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
/*! ZSTD_resetCStream() :
- * start a new compression job, using same parameters from previous job.
+ * start a new frame, using same parameters from previous frame.
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
* Note that zcs must be init at least once before using ZSTD_resetCStream().
* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
@@ -790,16 +1536,13 @@ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx
* + there is no active job (could be checked with ZSTD_frameProgression()), or
* + oldest job is still actively compressing data,
* but everything it has produced has also been flushed so far,
- * therefore flushing speed is currently limited by production speed of oldest job
- * irrespective of the speed of concurrent newer jobs.
+ * therefore flush speed is limited by production speed of oldest job
+ * irrespective of the speed of concurrent (and newer) jobs.
*/
ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
-
/*===== Advanced Streaming decompression functions =====*/
-typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
-ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
@@ -940,12 +1683,17 @@ typedef struct {
unsigned dictID;
unsigned checksumFlag;
} ZSTD_frameHeader;
+
/** ZSTD_getFrameHeader() :
* decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+/*! ZSTD_getFrameHeader_advanced() :
+ * same as ZSTD_getFrameHeader(),
+ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
@@ -962,564 +1710,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-/* ============================================ */
-/** New advanced API (experimental) */
-/* ============================================ */
-
-/* API design :
- * In this advanced API, parameters are pushed one by one into an existing context,
- * using ZSTD_CCtx_set*() functions.
- * Pushed parameters are sticky : they are applied to next job, and any subsequent job.
- * It's possible to reset parameters to "default" using ZSTD_CCtx_reset().
- * Important : "sticky" parameters only work with `ZSTD_compress_generic()` !
- * For any other entry point, "sticky" parameters are ignored !
- *
- * This API is intended to replace all others advanced / experimental API entry points.
- */
-
-/* note on enum design :
- * All enum will be pinned to explicit values before reaching "stable API" status */
-
-typedef enum {
- /* Opened question : should we have a format ZSTD_f_auto ?
- * Today, it would mean exactly the same as ZSTD_f_zstd1.
- * But, in the future, should several formats become supported,
- * on the compression side, it would mean "default format".
- * On the decompression side, it would mean "automatic format detection",
- * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
- * Since meaning is a little different, another option could be to define different enums for compression and decompression.
- * This question could be kept for later, when there are actually multiple formats to support,
- * but there is also the question of pinning enum values, and pinning value `0` is especially important */
- ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
- ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number.
- * Useful to save 4 bytes per generated frame.
- * Decoder cannot recognise automatically this format, requiring instructions. */
-} ZSTD_format_e;
-
-typedef enum {
- /* Note: this enum and the behavior it controls are effectively internal
- * implementation details of the compressor. They are expected to continue
- * to evolve and should be considered only in the context of extremely
- * advanced performance tuning.
- *
- * Zstd currently supports the use of a CDict in two ways:
- *
- * - The contents of the CDict can be copied into the working context. This
- * means that the compression can search both the dictionary and input
- * while operating on a single set of internal tables. This makes
- * the compression faster per-byte of input. However, the initial copy of
- * the CDict's tables incurs a fixed cost at the beginning of the
- * compression. For small compressions (< 8 KB), that copy can dominate
- * the cost of the compression.
- *
- * - The CDict's tables can be used in-place. In this model, compression is
- * slower per input byte, because the compressor has to search two sets of
- * tables. However, this model incurs no start-up cost (as long as the
- * working context's tables can be reused). For small inputs, this can be
- * faster than copying the CDict's tables.
- *
- * Zstd has a simple internal heuristic that selects which strategy to use
- * at the beginning of a compression. However, if experimentation shows that
- * Zstd is making poor choices, it is possible to override that choice with
- * this enum.
- */
- ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
- ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
- ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
-} ZSTD_dictAttachPref_e;
-
-typedef enum {
- /* compression format */
- ZSTD_p_format = 10, /* See ZSTD_format_e enum definition.
- * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
-
- /* compression parameters */
- ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
- * Default level is ZSTD_CLEVEL_DEFAULT==3.
- * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
- * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type.
- * Note 2 : setting a level sets all default values of other compression parameters.
- * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */
- ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
- * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
- * Special: value 0 means "use default windowLog".
- * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
- * requires explicitly allowing such window size during decompression stage. */
- ZSTD_p_hashLog, /* Size of the initial probe table, as a power of 2.
- * Resulting table size is (1 << (hashLog+2)).
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
- * Larger tables improve compression ratio of strategies <= dFast,
- * and improve speed of strategies > dFast.
- * Special: value 0 means "use default hashLog". */
- ZSTD_p_chainLog, /* Size of the multi-probe search table, as a power of 2.
- * Resulting table size is (1 << (chainLog+2)).
- * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
- * Larger tables result in better and slower compression.
- * This parameter is useless when using "fast" strategy.
- * Note it's still useful when using "dfast" strategy,
- * in which case it defines a secondary probe table.
- * Special: value 0 means "use default chainLog". */
- ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
- * More attempts result in better and slower compression.
- * This parameter is useless when using "fast" and "dFast" strategies.
- * Special: value 0 means "use default searchLog". */
- ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
- * Larger values make faster compression and decompression, but decrease ratio.
- * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
- * Note that currently, for all strategies < btopt, effective minimum is 4.
- * , for all strategies > fast, effective maximum is 6.
- * Special: value 0 means "use default minMatchLength". */
- ZSTD_p_targetLength, /* Impact of this field depends on strategy.
- * For strategies btopt & btultra:
- * Length of Match considered "good enough" to stop search.
- * Larger values make compression stronger, and slower.
- * For strategy fast:
- * Distance between match sampling.
- * Larger values make compression faster, and weaker.
- * Special: value 0 means "use default targetLength". */
- ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
- * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
- * The higher the value of selected strategy, the more complex it is,
- * resulting in stronger and slower compression.
- * Special: value 0 means "use default strategy". */
-
- ZSTD_p_enableLongDistanceMatching=160, /* Enable long distance matching.
- * This parameter is designed to improve compression ratio
- * for large inputs, by finding large matches at long distance.
- * It increases memory usage and window size.
- * Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB
- * except when expressly set to a different value. */
- ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2.
- * Larger values increase memory usage and compression ratio,
- * but decrease compression speed.
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
- * default: windowlog - 7.
- * Special: value 0 means "automatically determine hashlog". */
- ZSTD_p_ldmMinMatch, /* Minimum match size for long distance matcher.
- * Larger/too small values usually decrease compression ratio.
- * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
- * Special: value 0 means "use default value" (default: 64). */
- ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution.
- * Larger values improve collision resolution but decrease compression speed.
- * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX .
- * Special: value 0 means "use default value" (default: 3). */
- ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table.
- * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
- * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
- * Larger values improve compression speed.
- * Deviating far from default value will likely result in a compression ratio decrease.
- * Special: value 0 means "automatically determine hashEveryLog". */
-
- /* frame parameters */
- ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
- * Content size must be known at the beginning of compression,
- * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
- ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
- ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
-
- /* multi-threading parameters */
- /* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD).
- * They return an error otherwise. */
- ZSTD_p_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
- * When nbWorkers >= 1, triggers asynchronous mode :
- * ZSTD_compress_generic() consumes some input, flush some output if possible, and immediately gives back control to caller,
- * while compression work is performed in parallel, within worker threads.
- * (note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call).
- * More workers improve speed, but also increase memory usage.
- * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
- ZSTD_p_jobSize, /* Size of a compression job. This value is enforced only in non-blocking mode.
- * Each compression job is completed in parallel, so this value indirectly controls the nb of active threads.
- * 0 means default, which is dynamically determined based on compression parameters.
- * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
- * The minimum size is automatically and transparently enforced */
- ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
- * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-
- /* =================================================================== */
- /* experimental parameters - no stability guaranteed */
- /* =================================================================== */
-
- ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
- * even when referencing into Dictionary content (default:0) */
- ZSTD_p_forceAttachDict, /* Controls whether the contents of a CDict are
- * used in place, or whether they are copied into
- * the working context.
- *
- * Accepts values from the ZSTD_dictAttachPref_e
- * enum. See the comments on that enum for an
- * explanation of the feature.
- */
- ZSTD_p_rsyncable, /* Enables rsyncable mode, which makes compressed
- * files more rsync friendly by adding periodic
- * synchronization points to the compressed data.
- * The target average block size is
- * ZSTD_p_jobSize / 2. You can modify the job size
- * to increase or decrease the granularity of the
- * synchronization point. Once the jobSize is
- * smaller than the window size, you will start to
- * see degraded compression ratio.
- * NOTE: This only works when multithreading is
- * enabled.
- * NOTE: You probably don't want to use this with
- * long range mode, since that will decrease the
- * effectiveness of the synchronization points,
- * but your milage may vary.
- * NOTE: Rsyncable mode will limit the maximum
- * compression speed to approximately 400 MB/s.
- * If your compression level is already running
- * significantly slower than that (< 200 MB/s),
- * the speed won't be significantly impacted.
- */
-} ZSTD_cParameter;
-
-
-/*! ZSTD_CCtx_setParameter() :
- * Set one compression parameter, selected by enum ZSTD_cParameter.
- * Setting a parameter is generally only possible during frame initialization (before starting compression).
- * Exception : when using multi-threading mode (nbThreads >= 1),
- * following parameters can be updated _during_ compression (within same frame):
- * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
- * new parameters will be active on next job, or after a flush().
- * Note : when `value` type is not unsigned (int, or enum), cast it to unsigned for proper type checking.
- * @result : informational value (typically, value being set, correctly clamped),
- * or an error code (which can be tested with ZSTD_isError()). */
-ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
-
-/*! ZSTD_CCtx_getParameter() :
- * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value);
-
-/*! ZSTD_CCtx_setPledgedSrcSize() :
- * Total input data size to be compressed as a single frame.
- * This value will be controlled at the end, and result in error if not respected.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Note 1 : 0 means zero, empty.
- * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
- * Note 2 : If all data is provided and consumed in a single round,
- * this value is overriden by srcSize instead. */
-ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
-
-/*! ZSTD_CCtx_loadDictionary() :
- * Create an internal CDict from `dict` buffer.
- * Decompression will have to use same dictionary.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special: Adding a NULL (or 0-size) dictionary invalidates previous dictionary,
- * meaning "return to no-dictionary mode".
- * Note 1 : Dictionary will be used for all future compression jobs.
- * To return to "no-dictionary" situation, load a NULL dictionary
- * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
- * For this reason, compression parameters cannot be changed anymore after loading a dictionary.
- * It's also a CPU consuming operation, with non-negligible impact on latency.
- * Note 3 :`dict` content will be copied internally.
- * Use ZSTD_CCtx_loadDictionary_byReference() to reference dictionary content instead.
- * In such a case, dictionary buffer must outlive its users.
- * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
- * to precisely select how dictionary content must be interpreted. */
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
-
-
-/*! ZSTD_CCtx_refCDict() :
- * Reference a prepared dictionary, to be used for all next compression jobs.
- * Note that compression parameters are enforced from within CDict,
- * and supercede any compression parameter previously set within CCtx.
- * The dictionary will remain valid for future compression jobs using same CCtx.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special : adding a NULL CDict means "return to no-dictionary mode".
- * Note 1 : Currently, only one dictionary can be managed.
- * Adding a new dictionary effectively "discards" any previous one.
- * Note 2 : CDict is just referenced, its lifetime must outlive CCtx. */
-ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
-/*! ZSTD_CCtx_refPrefix() :
- * Reference a prefix (single-usage dictionary) for next compression job.
- * Decompression will need same prefix to properly regenerate data.
- * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
- * but performs much faster, especially during decompression (compression speed is tunable with compression level).
- * Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
- * Note 1 : Prefix buffer is referenced. It **must** outlive compression job.
- * Its contain must remain unmodified up to end of compression (ZSTD_e_end).
- * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
- * ensure that the window size is large enough to contain the entire source.
- * See ZSTD_p_windowLog.
- * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- * It's a CPU consuming operation, with non-negligible impact on latency.
- * If there is a need to use same prefix multiple times, consider loadDictionary instead.
- * Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
- * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
- const void* prefix, size_t prefixSize);
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
- const void* prefix, size_t prefixSize,
- ZSTD_dictContentType_e dictContentType);
-
-/*! ZSTD_CCtx_reset() :
- * Return a CCtx to clean state.
- * Useful after an error, or to interrupt an ongoing compression job and start a new one.
- * Any internal data not yet flushed is cancelled.
- * The parameters and dictionary are kept unchanged, to reset them use ZSTD_CCtx_resetParameters().
- */
-ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
-
-/*! ZSTD_CCtx_resetParameters() :
- * All parameters are back to default values (compression level is ZSTD_CLEVEL_DEFAULT).
- * Dictionary (if any) is dropped.
- * Resetting parameters is only possible during frame initialization (before starting compression).
- * To reset the context use ZSTD_CCtx_reset().
- * @return 0 or an error code (which can be checked with ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx);
-
-
-
-typedef enum {
- ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
- ZSTD_e_flush, /* flush any data provided so far,
- * it creates (at least) one new block, that can be decoded immediately on reception;
- * frame will continue: any future data can still reference previously compressed data, improving compression. */
- ZSTD_e_end /* flush any remaining data and close current frame.
- * any additional data starts a new frame.
- * each frame is independent (does not reference any content from previous frame). */
-} ZSTD_EndDirective;
-
-/*! ZSTD_compress_generic() :
- * Behave about the same as ZSTD_compressStream. To note :
- * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
- * - Compression parameters cannot be changed once compression is started.
- * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- * - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
- * - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
- * and then immediately returns, just indicating that there is some data remaining to be flushed.
- * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- * - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
- * - @return provides a minimum amount of data remaining to be flushed from internal buffers
- * or an error code, which can be tested using ZSTD_isError().
- * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
- * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
- * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
- * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
- * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
- * Before starting a new compression job, or changing compression parameters,
- * it is required to fully flush internal buffers.
- */
-ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input,
- ZSTD_EndDirective endOp);
-
-
-/*! ZSTD_compress_generic_simpleArgs() :
- * Same as ZSTD_compress_generic(),
- * but using only integral types as arguments.
- * Argument list is larger than ZSTD_{in,out}Buffer,
- * but can be helpful for binders from dynamic languages
- * which have troubles handling structures containing memory pointers.
- */
-ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs (
- ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos,
- ZSTD_EndDirective endOp);
-
-
-/*! ZSTD_CCtx_params :
- * Quick howto :
- * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
- * an existing ZSTD_CCtx_params structure.
- * This is similar to
- * ZSTD_CCtx_setParameter().
- * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
- * an existing CCtx.
- * These parameters will be applied to
- * all subsequent compression jobs.
- * - ZSTD_compress_generic() : Do compression using the CCtx.
- * - ZSTD_freeCCtxParams() : Free the memory.
- *
- * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
- * for static allocation for single-threaded compression.
- */
-ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
-ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
-
-
-/*! ZSTD_CCtxParams_reset() :
- * Reset params to default values.
- */
-ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
-
-/*! ZSTD_CCtxParams_init() :
- * Initializes the compression parameters of cctxParams according to
- * compression level. All other parameters are reset to their default values.
- */
-ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
-
-/*! ZSTD_CCtxParams_init_advanced() :
- * Initializes the compression and frame parameters of cctxParams according to
- * params. All other parameters are reset to their default values.
- */
-ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
-
-
-/*! ZSTD_CCtxParam_setParameter() :
- * Similar to ZSTD_CCtx_setParameter.
- * Set one compression parameter, selected by enum ZSTD_cParameter.
- * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
- * Note : when `value` is an enum, cast it to unsigned for proper type checking.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
-
-/*! ZSTD_CCtxParam_getParameter() :
- * Similar to ZSTD_CCtx_getParameter.
- * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned* value);
-
-/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
- * Apply a set of ZSTD_CCtx_params to the compression context.
- * This can be done even after compression is started,
- * if nbWorkers==0, this will have no impact until a new compression is started.
- * if nbWorkers>=1, new parameters will be picked up at next job,
- * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
- */
-ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
- ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
-
-
-/* ==================================== */
-/*=== Advanced decompression API ===*/
-/* ==================================== */
-
-/* The following API works the same way as the advanced compression API :
- * a context is created, parameters are pushed into it one by one,
- * then the context can be used to decompress data using an interface similar to the straming API.
- */
-
-/*! ZSTD_DCtx_loadDictionary() :
- * Create an internal DDict from dict buffer,
- * to be used to decompress next frames.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
- * meaning "return to no-dictionary mode".
- * Note 1 : `dict` content will be copied internally.
- * Use ZSTD_DCtx_loadDictionary_byReference()
- * to reference dictionary content instead.
- * In which case, the dictionary buffer must outlive its users.
- * Note 2 : Loading a dictionary involves building tables,
- * which has a non-negligible impact on CPU usage and latency.
- * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
- * how dictionary content will be interpreted and loaded.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
-
-
-/*! ZSTD_DCtx_refDDict() :
- * Reference a prepared dictionary, to be used to decompress next frames.
- * The dictionary remains active for decompression of future frames using same DCtx.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Note 1 : Currently, only one dictionary can be managed.
- * Referencing a new dictionary effectively "discards" any previous one.
- * Special : adding a NULL DDict means "return to no-dictionary mode".
- * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
-
-
-/*! ZSTD_DCtx_refPrefix() :
- * Reference a prefix (single-usage dictionary) for next compression job.
- * This is the reverse operation of ZSTD_CCtx_refPrefix(),
- * and must use the same prefix as the one used during compression.
- * Prefix is **only used once**. Reference is discarded at end of frame.
- * End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
- * Note 2 : Prefix buffer is referenced. It **must** outlive decompression job.
- * Prefix buffer must remain unmodified up to the end of frame,
- * reached when ZSTD_DCtx_decompress_generic() returns 0.
- * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
- * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
- * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
- * A fulldict prefix is more costly though.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
- const void* prefix, size_t prefixSize);
-ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
- const void* prefix, size_t prefixSize,
- ZSTD_dictContentType_e dictContentType);
-
-
-/*! ZSTD_DCtx_setMaxWindowSize() :
- * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
- * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
- * This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
- * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
- * @return : 0, or an error code (which can be tested using ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
-
-
-/*! ZSTD_DCtx_setFormat() :
- * Instruct the decoder context about what kind of data to decode next.
- * This instruction is mandatory to decode data without a fully-formed header,
- * such ZSTD_f_zstd1_magicless for example.
- * @return : 0, or an error code (which can be tested using ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
-
-
-/*! ZSTD_getFrameHeader_advanced() :
- * same as ZSTD_getFrameHeader(),
- * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr,
- const void* src, size_t srcSize, ZSTD_format_e format);
-
-
-/*! ZSTD_decompress_generic() :
- * Behave the same as ZSTD_decompressStream.
- * Decompression parameters cannot be changed once decompression is started.
- * @return : an error code, which can be tested using ZSTD_isError()
- * if >0, a hint, nb of expected input bytes for next invocation.
- * `0` means : a frame has just been fully decoded and flushed.
- */
-ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input);
-
-
-/*! ZSTD_decompress_generic_simpleArgs() :
- * Same as ZSTD_decompress_generic(),
- * but using only integral types as arguments.
- * Argument list is larger than ZSTD_{in,out}Buffer,
- * but can be helpful for binders from dynamic languages
- * which have troubles handling structures containing memory pointers.
- */
-ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs (
- ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos);
-
-
-/*! ZSTD_DCtx_reset() :
- * Return a DCtx to clean state.
- * If a decompression was ongoing, any internal data not yet flushed is cancelled.
- * All parameters are back to default values, including sticky ones.
- * Dictionary (if any) is dropped.
- * Parameters can be modified again after a reset.
- */
-ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
-
-
/* ============================ */
/** Block level API */
@@ -1539,10 +1729,10 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
+ copyCCtx() and copyDCtx() can be used too
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
- + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
+ + For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
- In which case, nothing is produced into `dst`.
+ In which case, nothing is produced into `dst` !
+ User must test for such outcome and deal directly with uncompressed data
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
+ In case of multiple successive blocks, should some of them be uncompressed,
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 5c3de10c..af2fb6ad 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -133,7 +133,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
0, /* ldmMinMatch */
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
- 0 /* ldmHashEveryLog */
+ 0 /* ldmHashRateLog */
};
return res;
}
@@ -160,32 +160,31 @@ typedef struct {
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
- ZSTD_CCtx_reset(ctx);
- ZSTD_CCtx_resetParameters(ctx);
+ ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
if (adv->nbWorkers==1) {
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0);
} else {
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers);
}
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength);
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) {
- ZSTD_DCtx_reset(dctx);
+ ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
}
@@ -224,22 +223,8 @@ static size_t local_defaultCompress(
void* dstBuffer, size_t dstSize,
void* addArgs)
{
- size_t moreToFlush = 1;
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
- ZSTD_inBuffer in;
- ZSTD_outBuffer out;
- in.src = srcBuffer; in.size = srcSize; in.pos = 0;
- out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
- while (moreToFlush) {
- if(out.pos == out.size) {
- return (size_t)-ZSTD_error_dstSize_tooSmall;
- }
- moreToFlush = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
- if (ZSTD_isError(moreToFlush)) {
- return moreToFlush;
- }
- }
- return out.pos;
+ return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
}
/* `addArgs` is the context */
@@ -258,7 +243,7 @@ static size_t local_defaultDecompress(
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
- moreToFlush = ZSTD_decompress_generic(dctx, &out, &in);
+ moreToFlush = ZSTD_decompressStream(dctx, &out, &in);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index 9860adf1..3a8b893e 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -115,7 +115,7 @@ typedef struct {
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
unsigned ldmHashLog;
unsigned ldmBucketSizeLog;
- unsigned ldmHashEveryLog;
+ unsigned ldmHashRateLog;
} BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */
diff --git a/programs/fileio.c b/programs/fileio.c
index 2818b96e..07f08539 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -346,9 +346,9 @@ void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
}
-static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET;
-void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
- g_ldmHashEveryLog = ldmHashEveryLog;
+static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
+void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) {
+ g_ldmHashRateLog = ldmHashRateLog;
}
@@ -529,42 +529,42 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) );
/* compression level */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)cLevel) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
/* long distance matching */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) );
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) );
}
- if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
+ if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) );
}
/* compression parameters */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams.windowLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams.chainLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams.hashLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams.searchLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams.searchLength) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams.targetLength) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams.strategy) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionStrategy, comprParams.strategy) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbWorkers, g_nbWorkers) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_jobSize, g_blockSize) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) );
if ( (g_overlapLog == FIO_OVERLAP_LOG_NOTSET)
&& (cLevel == ZSTD_maxCLevel()) )
g_overlapLog = 9; /* full overlap */
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog);
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_overlapSizeLog, g_overlapLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapSizeLog, g_overlapLog) );
}
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_rsyncable, g_rsyncable) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) );
#endif
/* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */
@@ -884,7 +884,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
size_t const oldIPos = inBuff.pos;
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
- CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
+ CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
/* count stats */
inputPresented++;
@@ -994,14 +994,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */
- ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
+ ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --;
if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */
- ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
+ ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
speedChange = noChange;
@@ -1313,7 +1313,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
- CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) );
+ CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) );
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize();
@@ -2111,7 +2111,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
- if (numBytesRead < ZSTD_frameHeaderSize_min) {
+ if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) {
if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0)
@@ -2172,7 +2172,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
info->numActualFrames++;
}
/* Skippable frame */
- else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
diff --git a/programs/fileio.h b/programs/fileio.h
index 8edb7dfe..7e1b1cd7 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -56,7 +56,7 @@ void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void FIO_setLdmFlag(unsigned ldmFlag);
-void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
+void FIO_setLdmHashRateLog(unsigned ldmHashRateLog);
void FIO_setLdmHashLog(unsigned ldmHashLog);
void FIO_setLdmMinMatch(unsigned ldmMinMatch);
void FIO_setMemLimit(unsigned memLimit);
diff --git a/programs/zstd.1 b/programs/zstd.1
index 5c2443ac..c6ae2c38 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -127,6 +127,10 @@ Does not spawn a thread for compression, use a single thread for both I/O and co
\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.
.TP
+\fB\-\-rsyncable\fR
+\fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
+.
+.TP
\fB\-D file\fR
use \fBfile\fR as Dictionary to compress or decompress FILE(s)
.
@@ -355,14 +359,14 @@ More searches increases the chance to find a match which usually increases compr
The minimum \fIslog\fR is 1 and the maximum is 26\.
.
.TP
-\fBsearchLength\fR=\fIslen\fR, \fBslen\fR=\fIslen\fR
+\fBminMatch\fR=\fImml\fR, \fBmml\fR=\fImml\fR
Specify the minimum searched length of a match in a hash table\.
.
.IP
Larger search lengths usually decrease compression ratio but improve decompression speed\.
.
.IP
-The minimum \fIslen\fR is 3 and the maximum is 7\.
+The minimum \fImml\fR is 3 and the maximum is 7\.
.
.TP
\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
@@ -388,7 +392,7 @@ Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This pa
The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
.
.TP
-\fBldmHashLog\fR=\fIldmhlog\fR, \fBldmhlog\fR=\fIldmhlog\fR
+\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
Specify the maximum size for a hash table used for long distance matching\.
.
.IP
@@ -398,10 +402,10 @@ This option is ignored unless long distance matching is enabled\.
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
.
.IP
-The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
+The minimum \fIlhlog\fR is 6 and the maximum is 26 (default: 20)\.
.
.TP
-\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
+\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
Specify the minimum searched length of a match for long distance matching\.
.
.IP
@@ -411,10 +415,10 @@ This option is ignored unless long distance matching is enabled\.
Larger/very small values usually decrease compression ratio\.
.
.IP
-The minimum \fIldmslen\fR is 4 and the maximum is 4096 (default: 64)\.
+The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 64)\.
.
.TP
-\fBldmBucketSizeLog\fR=\fIldmblog\fR, \fBldmblog\fR=\fIldmblog\fR
+\fBldmBucketSizeLog\fR=\fIlblog\fR, \fBlblog\fR=\fIlblog\fR
Specify the size of each bucket for the hash table used for long distance matching\.
.
.IP
@@ -424,10 +428,10 @@ This option is ignored unless long distance matching is enabled\.
Larger bucket sizes improve collision resolution but decrease compression speed\.
.
.IP
-The minimum \fIldmblog\fR is 0 and the maximum is 8 (default: 3)\.
+The minimum \fIlblog\fR is 0 and the maximum is 8 (default: 3)\.
.
.TP
-\fBldmHashEveryLog\fR=\fIldmhevery\fR, \fBldmhevery\fR=\fIldmhevery\fR
+\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
Specify the frequency of inserting entries into the long distance matching hash table\.
.
.IP
@@ -437,13 +441,13 @@ This option is ignored unless long distance matching is enabled\.
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
.
.IP
-The default value is \fBwlog \- ldmhlog\fR\.
+The default value is \fBwlog \- lhlog\fR\.
.
.SS "Example"
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
.
.P
-\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
+\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.
.SS "\-B#:"
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 4920ac01..f6a6e2bd 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -383,13 +383,13 @@ The list of available _options_:
The minimum _slog_ is 1 and the maximum is 26.
-- `searchLength`=_slen_, `slen`=_slen_:
+- `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table.
Larger search lengths usually decrease compression ratio but improve
decompression speed.
- The minimum _slen_ is 3 and the maximum is 7.
+ The minimum _mml_ is 3 and the maximum is 7.
- `targetLen`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
@@ -420,7 +420,7 @@ The list of available _options_:
Default _ovlog_ is 6, which means "reload `windowSize / 8`".
Exception : the maximum compression level (22) has a default _ovlog_ of 9.
-- `ldmHashLog`=_ldmhlog_, `ldmhlog`=_ldmhlog_:
+- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
Specify the maximum size for a hash table used for long distance matching.
This option is ignored unless long distance matching is enabled.
@@ -428,18 +428,18 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
- The minimum _ldmhlog_ is 6 and the maximum is 26 (default: 20).
+ The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
-- `ldmSearchLength`=_ldmslen_, `ldmslen`=_ldmslen_:
+- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
This option is ignored unless long distance matching is enabled.
Larger/very small values usually decrease compression ratio.
- The minimum _ldmslen_ is 4 and the maximum is 4096 (default: 64).
+ The minimum _lmml_ is 4 and the maximum is 4096 (default: 64).
-- `ldmBucketSizeLog`=_ldmblog_, `ldmblog`=_ldmblog_:
+- `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_:
Specify the size of each bucket for the hash table used for long distance
matching.
@@ -448,9 +448,9 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression
speed.
- The minimum _ldmblog_ is 0 and the maximum is 8 (default: 3).
+ The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
-- `ldmHashEveryLog`=_ldmhevery_, `ldmhevery`=_ldmhevery_:
+- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
hash table.
@@ -459,13 +459,13 @@ The list of available _options_:
Larger values will improve compression speed. Deviating far from the
default value will likely result in a decrease in compression ratio.
- The default value is `wlog - ldmhlog`.
+ The default value is `wlog - lhlog`.
### Example
The following parameters sets advanced compression options to something
similar to predefined level 19 for files bigger than 256 KB:
-`--zstd`=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
+`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#:
Select the size of each compression job.
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 9f9bc842..57440e3c 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -81,7 +81,7 @@ static const unsigned g_defaultMaxWindowLog = 27;
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
static U32 g_ldmHashLog = 0;
static U32 g_ldmMinMatch = 0;
-static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
+static U32 g_ldmHashRateLog = LDM_PARAM_DEFAULT;
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
@@ -392,7 +392,7 @@ static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, in
/** parseCompressionParameters() :
- * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
+ * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6") into *params
* @return 1 means that compression parameters were correct
* @return 0 in case of malformed parameters
*/
@@ -403,20 +403,20 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "minMatch=") || longCommandWArg(&stringPtr, "mml=")) { params->minMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "lhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmMinMatch=") || longCommandWArg(&stringPtr, "lmml=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "lblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmHashRateLog=") || longCommandWArg(&stringPtr, "lhrlog=")) { g_ldmHashRateLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
DISPLAYLEVEL(4, "invalid compression parameter \n");
return 0;
}
DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
- DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
+ DISPLAYLEVEL(4, "minMatch=%d, targetLength=%d, strategy=%d \n", params->minMatch, params->targetLength, params->strategy);
if (stringPtr[0] != 0) return 0; /* check the end of string */
return 1;
}
@@ -940,8 +940,8 @@ int main(int argCount, const char* argv[])
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
}
- if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
- benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
+ if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
+ benchParams.ldmHashRateLog = g_ldmHashRateLog;
}
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
@@ -1051,7 +1051,7 @@ int main(int argCount, const char* argv[])
FIO_setLdmHashLog(g_ldmHashLog);
FIO_setLdmMinMatch(g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
- if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
+ if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog);
FIO_setAdaptiveMode(adapt);
FIO_setAdaptMin(adaptMin);
FIO_setAdaptMax(adaptMax);
diff --git a/tests/README.md b/tests/README.md
index f28766bd..7c6fb0db 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -41,7 +41,7 @@ Additional remarks:
The example usage with two test files, one e-mail address, and with an additional message:
```
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
-```
+```
To run the script in background please use:
```
@@ -100,19 +100,19 @@ Full list of arguments
h# - hashLog
c# - chainLog
s# - searchLog
- l# - searchLength
+ l# - minMatch
t# - targetLength
S# - strategy
L# - level
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
- (Added forceAttachDictionary / fadt)
- When invoked with --optimize, this represents the sample to exceed.
+ (Added forceAttachDictionary / fadt)
+ When invoked with --optimize, this represents the sample to exceed.
--optimize= : find parameters to maximize compression ratio given parameters
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
cSpeed= : Minimum compression speed
dSpeed= : Minimum decompression speed
cMem= : Maximum compression memory
- lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
+ lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
(Lower value will begin with stronger strategies) (Default 90%)
@@ -121,13 +121,13 @@ Full list of arguments
when determining overall winner (default 5 (1% ratio = 5% speed)).
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
Higher values will make optimizer run longer, more chances to find better solution.
- memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
- Setting memLog = 0 turns off memoization
+ memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
+ Setting memLog = 0 turns off memoization
--display= : specifiy which parameters are included in the output
- can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
+ can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
(Default: display all params available)
-P# : generated sample compressibility (when no file is provided)
- -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
+ -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
-v : Prints Benchmarking output
-D : Next argument dictionary file
-s : Benchmark all files separately
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 409de639..33a91d70 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -171,17 +171,8 @@ local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{
- ZSTD_outBuffer buffOut;
- ZSTD_inBuffer buffIn;
(void)buff2;
- buffOut.dst = dst;
- buffOut.size = dstCapacity;
- buffOut.pos = 0;
- buffIn.src = src;
- buffIn.size = srcSize;
- buffIn.pos = 0;
- ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
- return buffOut.pos;
+ return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
@@ -198,8 +189,8 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
- ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
- ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
+ ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
+ ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
return buffOut.pos;
}
@@ -208,18 +199,9 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{
- ZSTD_outBuffer buffOut;
- ZSTD_inBuffer buffIn;
(void)buff2;
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
- buffOut.dst = dst;
- buffOut.size = dstCapacity;
- buffOut.pos = 0;
- buffIn.src = src;
- buffIn.size = srcSize;
- buffIn.pos = 0;
- while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
- return buffOut.pos;
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
+ return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
@@ -230,15 +212,15 @@ local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
- ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
- while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
+ ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
+ while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
return buffOut.pos;
}
@@ -408,28 +390,28 @@ static size_t benchMem(U32 benchNb,
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
- /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n",
+ /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d mml %d tlen %d strat %d \n",
cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
- cparams->searchLength, cparams->targetLength, cparams->strategy); */
+ cparams->minMatch, cparams->targetLength, cparams->strategy); */
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams.windowLog);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams.hashLog);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams.chainLog);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams.searchLog);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams.searchLength);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams.targetLength);
- ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams.strategy);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
+ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionStrategy, cparams.strategy);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams.windowLog);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams.hashLog);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams.chainLog);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams.searchLog);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams.searchLength);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams.targetLength);
- ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams.strategy);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
+ ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionStrategy, cparams.strategy);
/* Preparation */
switch(benchNb)
@@ -455,8 +437,8 @@ static size_t benchMem(U32 benchNb,
ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
- frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
- if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
+ frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
+ if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
@@ -476,8 +458,8 @@ static size_t benchMem(U32 benchNb,
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
- frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
- if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
+ frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
+ if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
@@ -752,7 +734,7 @@ int main(int argc, const char** argv)
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
- if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
+ if (longCommandWArg(&argument, "minMatch=") || longCommandWArg(&argument, "mml=")) { cparams.minMatch = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c
index 0921106d..83608b6e 100644
--- a/tests/fuzz/simple_round_trip.c
+++ b/tests/fuzz/simple_round_trip.c
@@ -40,9 +40,9 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
size_t err;
- ZSTD_CCtx_reset(cctx);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
FUZZ_setRandomParameters(cctx, srcSize, &seed);
- err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+ err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(err);
FUZZ_ASSERT(err == 0);
cSize = out.pos;
diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index 72d70495..d903bcb2 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -56,7 +56,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
const uint8_t *src, size_t srcSize)
{
size_t dstSize = 0;
- ZSTD_CCtx_reset(cctx);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
FUZZ_setRandomParameters(cctx, srcSize, &seed);
while (srcSize > 0) {
@@ -72,7 +72,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
case 1: /* fall-though */
case 2: {
size_t const ret =
- ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
+ ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
FUZZ_ZASSERT(ret);
if (ret == 0)
mode = -1;
@@ -80,11 +80,11 @@ static size_t compress(uint8_t *dst, size_t capacity,
}
case 3: {
size_t ret =
- ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+ ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
/* Reset the compressor when the frame is finished */
if (ret == 0) {
- ZSTD_CCtx_reset(cctx);
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
if ((FUZZ_rand(&seed) & 7) == 0) {
size_t const remaining = in.size - in.pos;
FUZZ_setRandomParameters(cctx, remaining, &seed);
@@ -95,7 +95,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
}
default: {
size_t const ret =
- ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
+ ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
FUZZ_ZASSERT(ret);
mode = -1;
}
@@ -108,7 +108,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
for (;;) {
ZSTD_inBuffer in = {NULL, 0, 0};
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
- size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
dst += out.pos;
diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c
index 75d0359a..9c17b3dc 100644
--- a/tests/fuzz/zstd_helpers.c
+++ b/tests/fuzz/zstd_helpers.c
@@ -32,8 +32,8 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
- cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
- ZSTD_SEARCHLENGTH_MAX);
+ cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
+ ZSTD_MINMATCH_MAX);
cParams.targetLength = FUZZ_rand32(state, 0, 512);
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
return ZSTD_adjustCParams(cParams, srcSize, 0);
@@ -60,25 +60,25 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
{
ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
- set(cctx, ZSTD_p_windowLog, cParams.windowLog);
- set(cctx, ZSTD_p_hashLog, cParams.hashLog);
- set(cctx, ZSTD_p_chainLog, cParams.chainLog);
- set(cctx, ZSTD_p_searchLog, cParams.searchLog);
- set(cctx, ZSTD_p_minMatch, cParams.searchLength);
- set(cctx, ZSTD_p_targetLength, cParams.targetLength);
- set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
+ set(cctx, ZSTD_c_windowLog, cParams.windowLog);
+ set(cctx, ZSTD_c_hashLog, cParams.hashLog);
+ set(cctx, ZSTD_c_chainLog, cParams.chainLog);
+ set(cctx, ZSTD_c_searchLog, cParams.searchLog);
+ set(cctx, ZSTD_c_minMatch, cParams.minMatch);
+ set(cctx, ZSTD_c_targetLength, cParams.targetLength);
+ set(cctx, ZSTD_c_compressionStrategy, cParams.strategy);
/* Select frame parameters */
- setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
- setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
- setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
- setRand(cctx, ZSTD_p_forceAttachDict, 0, 2, state);
+ setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
+ setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
+ setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
+ setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
/* Select long distance matchig parameters */
- setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
- setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
- setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
+ setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
+ setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
+ setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
ZSTD_LDM_MINMATCH_MAX, state);
- setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
+ setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
state);
- setRand(cctx, ZSTD_p_ldmHashEveryLog, 0,
- ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state);
+ setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
+ ZSTD_LDM_HASHRATELOG_MAX, state);
}
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 5d5c73e1..b5872c68 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -233,11 +233,9 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
mallocCounter_t malcount = INIT_MALLOC_COUNTER;
ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
- ZSTD_outBuffer out = { outBuffer, outSize, 0 };
- ZSTD_inBuffer in = { inBuffer, inSize, 0 };
- CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) );
- CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) );
- while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
+ CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
+ CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
+ CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
ZSTD_freeCCtx(cctx);
DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
nbThreads, compressionLevel);
@@ -255,10 +253,10 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
ZSTD_outBuffer out = { outBuffer, outSize, 0 };
ZSTD_inBuffer in = { inBuffer, inSize, 0 };
- CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) );
- CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) );
- CHECK_Z( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue) );
- while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
+ CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
+ CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
+ CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
+ while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
ZSTD_freeCCtx(cctx);
DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
nbThreads, compressionLevel);
@@ -311,7 +309,6 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
void* const compressedBuffer = malloc(compressedBufferSize);
void* const decodedBuffer = malloc(CNBuffSize);
- ZSTD_DCtx* dctx = ZSTD_createDCtx();
int testResult = 0;
U32 testNb=0;
size_t cSize;
@@ -383,13 +380,27 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
- { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
- if (r != CNBuffSize) goto _output_error; }
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
+ { size_t const r = ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ NULL, 0);
+ if (r != CNBuffSize) goto _output_error;
+ }
+ ZSTD_freeDCtx(dctx);
+ }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
- { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
- if (r != CNBuffSize) goto _output_error; }
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
+ { size_t const r = ZSTD_decompress_usingDDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ NULL);
+ if (r != CNBuffSize) goto _output_error;
+ }
+ ZSTD_freeDCtx(dctx);
+ }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
@@ -504,39 +515,39 @@ static int basicUnitTests(U32 seed, double compressibility)
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_outBuffer out = {NULL, 0, 0};
ZSTD_inBuffer in = {NULL, 0, 0};
- unsigned value;
+ int value;
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
CHECK_EQ(value, 3);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, 0);
- CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN));
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
CHECK_EQ(value, 3);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
- CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 7));
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
CHECK_EQ(value, 7);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
/* Start a compression job */
- ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
+ ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
CHECK_EQ(value, 7);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
/* Reset the CCtx */
- ZSTD_CCtx_reset(cctx);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
CHECK_EQ(value, 7);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
/* Reset the parameters */
- ZSTD_CCtx_resetParameters(cctx);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
CHECK_EQ(value, 3);
- CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
+ CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, 0);
ZSTD_freeCCtx(cctx);
@@ -546,8 +557,8 @@ static int basicUnitTests(U32 seed, double compressibility)
/* this test is really too long, and should be made faster */
DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
- ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
- size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */
+ ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+ size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
size_t cnb;
assert(cctx != NULL);
params.fParams.contentSizeFlag = 0;
@@ -753,7 +764,7 @@ static int basicUnitTests(U32 seed, double compressibility)
const U32 skipLen = 129 KB;
MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
- off += skipLen + ZSTD_skippableHeaderSize;
+ off += skipLen + ZSTD_SKIPPABLEHEADERSIZE;
}
}
cSize = off;
@@ -777,7 +788,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Dictionary and CCtx Duplication tests */
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t dictSize = 551;
+ assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
@@ -861,6 +874,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeCCtx(ctxOrig);
ZSTD_freeCCtx(ctxDuplicated);
+ ZSTD_freeDCtx(dctx);
}
/* Dictionary and dictBuilder tests */
@@ -905,7 +919,7 @@ static int basicUnitTests(U32 seed, double compressibility)
coverParams.nbThreads = 4;
dictSize = ZDICT_optimizeTrainFromBuffer_cover(
dictBuffer, dictBufferCapacity,
- CNBuffer, samplesSizes, nbSamples,
+ CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
&coverParams);
if (ZDICT_isError(dictSize)) goto _output_error;
}
@@ -950,11 +964,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
- CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
- decodedBuffer, CNBuffSize,
- compressedBuffer, cSize,
- dictBuffer, dictSize),
- if (r != CNBuffSize) goto _output_error);
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ ZSTD_freeDCtx(dctx);
+ }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
@@ -983,11 +1000,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
- CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
- decodedBuffer, CNBuffSize,
- compressedBuffer, cSize,
- dictBuffer, dictSize),
- if (r != CNBuffSize) goto _output_error);
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ ZSTD_freeDCtx(dctx);
+ }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
@@ -1036,11 +1056,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK (unknown)\n");
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
- CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
- decodedBuffer, CNBuffSize,
- compressedBuffer, cSize,
- dictBuffer, dictSize),
- if (r != CNBuffSize) goto _output_error);
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ ZSTD_freeDCtx(dctx);
+ }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
@@ -1054,19 +1077,26 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
- CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
- decodedBuffer, CNBuffSize,
- compressedBuffer, cSize,
- dictBuffer, dictSize),
- if (r != CNBuffSize) goto _output_error);
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ ZSTD_freeDCtx(dctx);
+ }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
- {
- const size_t ret = ZSTD_decompress_usingDict(
- dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
- "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
- if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error;
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ assert(dctx != NULL);
+ { const size_t ret = ZSTD_decompress_usingDict(
+ dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
+ "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
+ if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
+ goto _output_error;
+ }
+ ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1135,10 +1165,12 @@ static int basicUnitTests(U32 seed, double compressibility)
*/
{ size_t dSize;
BYTE data[1024];
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
ZSTD_dlm_byRef, ZSTD_dct_auto,
cParams, ZSTD_defaultCMem);
+ assert(dctx != NULL); assert(cdict != NULL);
memset(data, 'x', sizeof(data));
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
data, sizeof(data), cdict);
@@ -1147,6 +1179,7 @@ static int basicUnitTests(U32 seed, double compressibility)
dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
+ ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1252,13 +1285,13 @@ static int basicUnitTests(U32 seed, double compressibility)
params);
if (ZSTD_isError(cSize_1pass)) goto _output_error;
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel) );
- { ZSTD_inBuffer in = { CNBuffer, srcSize, 0 };
- ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
- size_t const compressionResult = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
- DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, out.pos);
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
+ { size_t const compressionResult = ZSTD_compress2(cctx,
+ compressedBuffer, compressedBufferSize,
+ CNBuffer, srcSize);
+ DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
if (ZSTD_isError(compressionResult)) goto _output_error;
- if (out.pos != cSize_1pass) goto _output_error;
+ if (compressionResult != cSize_1pass) goto _output_error;
} }
ZSTD_freeCCtx(cctx);
}
@@ -1271,16 +1304,15 @@ static int basicUnitTests(U32 seed, double compressibility)
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
assert(cctx != NULL);
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) );
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) );
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) );
- { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
- ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
- size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
- if (result != 0) goto _output_error;
- if (in.pos != in.size) goto _output_error;
- cSize = out.pos;
- xxh64 = XXH64(out.dst, out.pos, 0);
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
+ { size_t const compressedSize = ZSTD_compress2(cctx,
+ compressedBuffer, ZSTD_compressBound(inputSize),
+ CNBuffer, inputSize);
+ CHECK(compressedSize);
+ cSize = compressedSize;
+ xxh64 = XXH64(compressedBuffer, compressedSize, 0);
}
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize);
ZSTD_freeCCtx(cctx);
@@ -1288,32 +1320,65 @@ static int basicUnitTests(U32 seed, double compressibility)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) );
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) );
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) );
- { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
- ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
- size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
- if (result != 0) goto _output_error;
- if (in.pos != in.size) goto _output_error;
- if (out.pos != cSize) goto _output_error; /* must result in same compressed result, hence same size */
- if (XXH64(out.dst, out.pos, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
- DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)out.pos);
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
+ { size_t const result = ZSTD_compress2(cctx,
+ compressedBuffer, ZSTD_compressBound(inputSize),
+ CNBuffer, inputSize);
+ CHECK(result);
+ if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
+ if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
+ DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)result);
}
ZSTD_freeCCtx(cctx);
}
}
+ /* advanced parameters for decompression */
+ { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ assert(dctx != NULL);
+
+ DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
+ { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
+ CHECK(bounds.error);
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
+ DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
+ { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
+ if (!ZSTD_isError(sr)) goto _output_error;
+ }
+ { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
+ if (!ZSTD_isError(bounds.error)) goto _output_error;
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
+ DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
+ { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
+ if (!ZSTD_isError(sr)) goto _output_error;
+ }
+ { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
+ if (!ZSTD_isError(sr)) goto _output_error;
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
+ ZSTD_freeDCtx(dctx);
+ }
+
+
/* custom formats tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
+ assert(dctx != NULL); assert(cctx != NULL);
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
- CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) );
+ CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
- size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+ size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
cSize = out.pos;
@@ -1327,29 +1392,38 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
- ZSTD_DCtx_reset(dctx);
- CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) );
+ ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+ CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_frameHeader zfh;
size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
if (zfhrt != 0) goto _output_error;
}
+ /* one shot */
+ { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+ if (result != inputSize) goto _output_error;
+ DISPLAYLEVEL(3, "one-shot OK, ");
+ }
+ /* streaming */
{ ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
- size_t const result = ZSTD_decompress_generic(dctx, &out, &in);
+ size_t const result = ZSTD_decompressStream(dctx, &out, &in);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
if (out.pos != inputSize) goto _output_error;
- DISPLAYLEVEL(3, "OK : regenerated %u bytes \n", (U32)out.pos);
+ DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (U32)out.pos);
}
ZSTD_freeCCtx(cctx);
+ ZSTD_freeDCtx(dctx);
}
/* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t dictSize = 65 KB;
static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
size_t cSize2;
+ assert(cctx != NULL); assert(dctx != NULL);
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
@@ -1367,13 +1441,14 @@ static int basicUnitTests(U32 seed, double compressibility)
/* very long stream of block compression */
DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
- CHECK( ZSTD_compressBegin(cctx, -99) ); /* we just want to quickly overflow internal U32 index */
+ CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
{ U64 const toCompress = 5000000000ULL; /* > 4 GB */
U64 compressed = 0;
while (compressed < toCompress) {
size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
- if (ZSTD_isError(cSize)) goto _output_error;
+ assert(blockCSize != 0);
+ if (ZSTD_isError(blockCSize)) goto _output_error;
compressed += blockCSize;
}
}
@@ -1411,8 +1486,8 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeCCtx(cctx);
+ ZSTD_freeDCtx(dctx);
}
- ZSTD_freeDCtx(dctx);
/* long rle test */
{ size_t sampleSize = 0;
@@ -1927,7 +2002,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
/* ensure memory requirement is good enough (should always be true) */
{ ZSTD_frameHeader zfh;
- CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max),
+ CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
"ZSTD_getFrameHeader(): error retrieving frame information");
{ size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
CHECK_Z(roundBuffSize);
diff --git a/tests/longmatch.c b/tests/longmatch.c
index 1271e9ab..b673baa6 100644
--- a/tests/longmatch.c
+++ b/tests/longmatch.c
@@ -50,7 +50,7 @@ int main(int argc, const char** argv)
params.cParams.chainLog = 13;
params.cParams.hashLog = 14;
params.cParams.searchLog = 1;
- params.cParams.searchLength = 7;
+ params.cParams.minMatch = 7;
params.cParams.targetLength = 16;
params.cParams.strategy = ZSTD_fast;
windowLog = params.cParams.windowLog;
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index cd272d9a..21c2cfe6 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -75,7 +75,7 @@ static const int g_maxNbVariations = 64;
#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
-#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
+#define MML_RANGE (ZSTD_MINMATCH_MAX - ZSTD_MINMATCH_MIN + 1)
#define TLEN_RANGE 17
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
#define FADT_RANGE 3
@@ -103,7 +103,7 @@ typedef enum {
clog_ind = 1,
hlog_ind = 2,
slog_ind = 3,
- slen_ind = 4,
+ mml_ind = 4,
tlen_ind = 5,
strt_ind = 6,
fadt_ind = 7, /* forceAttachDict */
@@ -116,27 +116,27 @@ typedef struct {
/* maximum value of parameters */
static const U32 mintable[NUM_PARAMS] =
- { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
+ { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_MINMATCH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
/* minimum value of parameters */
static const U32 maxtable[NUM_PARAMS] =
- { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
+ { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_MINMATCH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
/* # of values parameters can take on */
static const U32 rangetable[NUM_PARAMS] =
- { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
+ { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, MML_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
/* ZSTD_cctxSetParameter() index to set */
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
- { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
+ { ZSTD_c_windowLog, ZSTD_c_chainLog, ZSTD_c_hashLog, ZSTD_c_searchLog, ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_compressionStrategy, ZSTD_c_forceAttachDict };
/* names of parameters */
static const char* g_paramNames[NUM_PARAMS] =
- { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
+ { "windowLog", "chainLog", "hashLog","searchLog", "minMatch", "targetLength", "strategy", "forceAttachDict" };
/* shortened names of parameters */
static const char* g_shortParamNames[NUM_PARAMS] =
- { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
+ { "wlog", "clog", "hlog", "slog", "mml", "tlen", "strat", "fadt" };
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
static U32 rangeMap(varInds_t param, int ind) {
@@ -150,7 +150,7 @@ static U32 rangeMap(varInds_t param, int ind) {
case clog_ind:
case hlog_ind:
case slog_ind:
- case slen_ind:
+ case mml_ind:
case strt_ind:
return mintable[param] + ind;
case NUM_PARAMS:
@@ -186,7 +186,7 @@ static int invRangeMap(varInds_t param, U32 value) {
case clog_ind:
case hlog_ind:
case slog_ind:
- case slen_ind:
+ case mml_ind:
case strt_ind:
return value - mintable[param];
case NUM_PARAMS:
@@ -205,7 +205,7 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
case clog_ind:
case hlog_ind:
case slog_ind:
- case slen_ind:
+ case mml_ind:
case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n "); break;
@@ -311,7 +311,7 @@ static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
c.chainLog = p.vals[clog_ind];
c.hashLog = p.vals[hlog_ind];
c.searchLog = p.vals[slog_ind];
- c.searchLength = p.vals[slen_ind];
+ c.minMatch = p.vals[mml_ind];
c.targetLength = p.vals[tlen_ind];
c.strategy = p.vals[strt_ind];
/* no forceAttachDict */
@@ -325,7 +325,7 @@ static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
p.vals[clog_ind] = c.chainLog;
p.vals[hlog_ind] = c.hashLog;
p.vals[slog_ind] = c.searchLog;
- p.vals[slen_ind] = c.searchLength;
+ p.vals[mml_ind] = c.minMatch;
p.vals[tlen_ind] = c.targetLength;
p.vals[strt_ind] = c.strategy;
@@ -893,9 +893,8 @@ typedef struct {
static size_t local_initCCtx(void* payload) {
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
varInds_t i;
- ZSTD_CCtx_reset(ag->cctx);
- ZSTD_CCtx_resetParameters(ag->cctx);
- ZSTD_CCtx_setParameter(ag->cctx, ZSTD_p_compressionLevel, ag->cLevel);
+ ZSTD_CCtx_reset(ag->cctx, ZSTD_reset_session_and_parameters);
+ ZSTD_CCtx_setParameter(ag->cctx, ZSTD_c_compressionLevel, ag->cLevel);
for(i = 0; i < NUM_PARAMS; i++) {
if(ag->comprParams->vals[i] != PARAM_UNSET)
@@ -914,37 +913,20 @@ typedef struct {
static size_t local_initDCtx(void* payload) {
const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
- ZSTD_DCtx_reset(ag->dctx);
+ ZSTD_DCtx_reset(ag->dctx, ZSTD_reset_session_and_parameters);
ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
return 0;
}
/* additional argument is just the context */
static size_t local_defaultCompress(
- const void* srcBuffer, size_t srcSize,
- void* dstBuffer, size_t dstSize,
- void* addArgs) {
- size_t moreToFlush = 1;
- ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
- ZSTD_inBuffer in;
- ZSTD_outBuffer out;
- in.src = srcBuffer;
- in.size = srcSize;
- in.pos = 0;
- out.dst = dstBuffer;
- out.size = dstSize;
- out.pos = 0;
+ const void* srcBuffer, size_t srcSize,
+ void* dstBuffer, size_t dstSize,
+ void* addArgs)
+{
+ ZSTD_CCtx* cctx = (ZSTD_CCtx*)addArgs;
assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */
- while (moreToFlush) {
- if(out.pos == out.size) {
- return (size_t)-ZSTD_error_dstSize_tooSmall;
- }
- moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
- if (ZSTD_isError(moreToFlush)) {
- return moreToFlush;
- }
- }
- return out.pos;
+ return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
}
/* additional argument is just the context */
@@ -966,7 +948,7 @@ static size_t local_defaultDecompress(
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
- moreToFlush = ZSTD_decompress_generic(dctx,
+ moreToFlush = ZSTD_decompressStream(dctx,
&out, &in);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
@@ -2665,7 +2647,7 @@ int main(int argc, const char** argv)
continue;
case 'l': /* search length */
argument++;
- g_params.vals[slen_ind] = readU32FromChar(&argument);
+ g_params.vals[mml_ind] = readU32FromChar(&argument);
continue;
case 't': /* target length */
argument++;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 99609a5e..456bef5c 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -239,11 +239,11 @@ $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wron
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
-roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
+roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
-roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
-roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7"
-roundTripTest -g512K " --single-thread --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7"
+roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
+roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
+roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
roundTripTest -g512K 19
diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c
index 90afcd4b..13cf5157 100644
--- a/tests/roundTripCrash.c
+++ b/tests/roundTripCrash.c
@@ -85,7 +85,7 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 };
- ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 };
+ ZSTD_outBuffer outBuffer = { compressedBuff, compressedBuffCapacity, 0 };
static const int maxClevel = 19;
size_t const hashLength = MIN(128, srcBuffSize);
@@ -93,15 +93,15 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
int const cLevel = h32 % maxClevel;
/* Set parameters */
- CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, cLevel) );
- CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbWorkers, 2) );
- CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, 5) );
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) );
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) );
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_overlapSizeLog, 5) );
/* Apply parameters */
CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
- CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
+ CHECK_Z (ZSTD_compressStream2(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
ZSTD_freeCCtxParams(cctxParams);
ZSTD_freeCCtx(cctx);
diff --git a/tests/symbols.c b/tests/symbols.c
index b3708213..600d8167 100644
--- a/tests/symbols.c
+++ b/tests/symbols.c
@@ -89,7 +89,6 @@ static const void *symbols[] = {
&ZSTD_sizeof_CStream,
&ZSTD_createDStream_advanced,
&ZSTD_initDStream_usingDict,
- &ZSTD_setDStreamParameter,
&ZSTD_initDStream_usingDDict,
&ZSTD_resetDStream,
&ZSTD_sizeof_DStream,
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 2e076d7b..d148b17c 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -175,11 +175,11 @@ static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
size_t cret;
do {
- ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
- ZSTD_inBuffer din = {compressed, 0, 0};
- ZSTD_outBuffer dout = {uncompressed, 0, 0};
+ ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
+ ZSTD_inBuffer din = { compressed, 0, 0 };
+ ZSTD_outBuffer dout = { uncompressed, 0, 0 };
- cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
+ cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
if (ZSTD_isError(cret))
return cret;
@@ -223,19 +223,19 @@ static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
{
- unsigned value;
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_windowLog, &savedParams->cParams.windowLog));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_hashLog, &savedParams->cParams.hashLog));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_chainLog, &savedParams->cParams.chainLog));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_searchLog, &savedParams->cParams.searchLog));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_minMatch, &savedParams->cParams.searchLength));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_targetLength, &savedParams->cParams.targetLength));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionStrategy, &value));
+ int value;
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionStrategy, &value));
savedParams->cParams.strategy = value;
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_checksumFlag, &savedParams->fParams.checksumFlag));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
- CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_dictIDFlag, &value));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
+ CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
savedParams->fParams.noDictIDFlag = !value;
return 0;
}
@@ -248,7 +248,7 @@ static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
- CHECK_RET(5, params.cParams.searchLength != savedParams.cParams.searchLength, "searchLength");
+ CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
@@ -353,7 +353,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
{ size_t r;
ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
- params.cParams.searchLength = 2;
+ params.cParams.minMatch = 2;
r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
if (!ZSTD_isError(r)) goto _output_error;
DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
@@ -379,7 +379,7 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff2 = inBuff;
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
- CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000) ); /* large limit */
+ CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
{ size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
@@ -649,16 +649,10 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
}
- /* test ZSTD_setDStreamParameter() resilience */
- DISPLAYLEVEL(3, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
- { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */
- if (!ZSTD_isError(r)) goto _output_error; }
- DISPLAYLEVEL(3, "OK \n");
-
/* Memory restriction */
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
- CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */
+ CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
outBuff.dst = decodedBuffer;
outBuff.size = CNBufferSize;
outBuff.pos = 0;
@@ -668,7 +662,7 @@ static int basicUnitTests(U32 seed, double compressibility)
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
- ZSTD_DCtx_reset(zd); /* leave zd in good shape for next tests */
+ ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
@@ -692,12 +686,12 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.size = size;
inBuff.pos = 0;
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
- CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end));
+ CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
if (inBuff.pos != inBuff.size) goto _output_error;
{ ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
- CHECK_Z( ZSTD_decompress_generic(dctx, &decOut, &decIn) );
+ CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
if (decIn.pos != decIn.size) goto _output_error;
if (decOut.pos != size) goto _output_error;
{ U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
@@ -752,7 +746,7 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
- CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
+ CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -765,7 +759,7 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
- CHECK_Z( ZSTD_decompress_generic(zd, &outBuff, &inBuff) );
+ CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
DISPLAYLEVEL(3, "OK \n");
@@ -776,14 +770,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
}
- DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++);
+ DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
- CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
+ CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -874,9 +868,9 @@ static int basicUnitTests(U32 seed, double compressibility)
const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
BYTE* const dst = (BYTE*)CNBuffer + start - offset;
DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
- CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) );
- CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbWorkers, nbWorkers) );
- CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) );
+ CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
+ CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
+ CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
assert(start > offset);
assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
@@ -891,7 +885,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
- CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
+ CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
ZSTD_freeCDict(cdict);
}
@@ -936,11 +930,11 @@ static int basicUnitTests(U32 seed, double compressibility)
if (!cdict || !ddict) goto _output_error;
- ZSTD_CCtx_reset(zc);
+ ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
ZSTD_resetDStream(zd);
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
- CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog));
+ CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
/* Test all values < 300 */
for (value = 0; value < 300; ++value) {
for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
@@ -968,12 +962,12 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
- { unsigned level;
+ { int level;
CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
- CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level));
+ CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
CHECK(level != 11, "Compression level does not match");
ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN);
- CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level));
+ CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
CHECK(level != 11, "Compression level does not match");
}
DISPLAYLEVEL(3, "OK \n");
@@ -1083,17 +1077,16 @@ static int basicUnitTests(U32 seed, double compressibility)
int remainingInput = 256 * 1024;
int offset;
- ZSTD_CCtx_reset(zc);
- CHECK_Z(ZSTD_CCtx_resetParameters(zc));
+ CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
- CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1));
+ CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
/* Write a bunch of 6 byte blocks */
while (remainingInput > 0) {
char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
const size_t kSmallBlockSize = sizeof(testBuffer);
ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
- CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush));
+ CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
CHECK(in.pos != in.size, "input not fully consumed");
remainingInput -= kSmallBlockSize;
}
@@ -1101,7 +1094,7 @@ static int basicUnitTests(U32 seed, double compressibility)
for (offset = 1024; offset >= 0; offset -= 128) {
ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
- CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));
+ CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
CHECK(in.pos != in.size, "input not fully consumed");
}
/* Ensure decompression works */
@@ -1824,7 +1817,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
{ int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
- CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_compressionLevel, compressionLevel, opaqueAPI) );
+ CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
}
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
@@ -1858,45 +1851,45 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
- cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
if (FUZ_rand(&lseed) & 1) {
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
}
if (FUZ_rand(&lseed) & 1) {
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
}
if (FUZ_rand(&lseed) & 1) {
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
}
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, opaqueAPI) );
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, opaqueAPI) );
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
/* mess with long distance matching parameters */
if (bigTests) {
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
}
/* mess with frame parameters */
if (FUZ_rand(&lseed) & 1) {
U32 const checksumFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, checksumFlag, opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
}
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
if (FUZ_rand(&lseed) & 1) {
DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
@@ -1908,17 +1901,17 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbWorkers, nbThreads, opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
if (nbThreads > 1) {
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapSizeLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
}
}
/* Enable rsyncable mode 1 in 4 times. */
- setCCtxParameter(zc, cctxParams, ZSTD_p_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI);
+ setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI);
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
/* Apply parameters */
if (opaqueAPI) {
@@ -1938,7 +1931,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams);
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed");
} else {
- size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1);
+ size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, cParams.windowLog-1);
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
}
}
@@ -1963,7 +1956,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
- CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
+ CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos), (U32)flush, (U32)outBuff.pos);
@@ -1980,10 +1973,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (U32)adjustedDstSize);
- remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
+ remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (U32)outBuff.pos);
CHECK( ZSTD_isError(remainingToFlush),
- "ZSTD_compress_generic w/ ZSTD_e_end error : %s",
+ "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
ZSTD_getErrorName(remainingToFlush) );
} }
crcOrig = XXH64_digest(&xxhState);
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index b3e5f367..0ee5a310 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -31,7 +31,7 @@
/* === Constants === */
#define Z_INFLATE_SYNC 8
#define ZLIB_HEADERSIZE 4
-#define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min
+#define ZSTD_HEADERSIZE ZSTD_FRAMEHEADERSIZE_MIN
#define ZWRAP_DEFAULT_CLEVEL 3 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
@@ -140,8 +140,8 @@ static int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dic
if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
{ ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
size_t initErr;
- LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n",
- (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy);
+ LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d minMatch=%d strategy=%d\n",
+ (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.minMatch, params.cParams.strategy);
initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(initErr)) return Z_STREAM_ERROR;
}