AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move posInSequence correction to correct location

Browse Source
This commit is contained in:
senhuang42 2020-10-06 16:16:41 -04:00
parent 0c515590d8
commit abce708a56
3 changed files with 83 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
doc/zstd_manual.html
View File

@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>zstd 1.4.5 Manual</title>
<title>zstd 1.4.6 Manual</title>
</head>
<body>
<h1>zstd 1.4.5 Manual</h1>
<h1>zstd 1.4.6 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@ -24,19 +24,20 @@
<li><a href="#Chapter14">experimental API (static linking only)</a></li>
<li><a href="#Chapter15">Frame size functions</a></li>
<li><a href="#Chapter16">Memory management</a></li>
<li><a href="#Chapter17">Advanced compression functions</a></li>
<li><a href="#Chapter18">Advanced decompression functions</a></li>
<li><a href="#Chapter19">Advanced streaming functions</a></li>
<li><a href="#Chapter20">! ZSTD_initCStream_usingDict() :</a></li>
<li><a href="#Chapter21">! ZSTD_initCStream_advanced() :</a></li>
<li><a href="#Chapter22">! ZSTD_initCStream_usingCDict() :</a></li>
<li><a href="#Chapter23">! ZSTD_initCStream_usingCDict_advanced() :</a></li>
<li><a href="#Chapter24">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter17">This API is temporary and is expected to change or disappear in the future!</a></li>
<li><a href="#Chapter18">Advanced compression functions</a></li>
<li><a href="#Chapter19">Advanced decompression functions</a></li>
<li><a href="#Chapter20">Advanced streaming functions</a></li>
<li><a href="#Chapter21">! ZSTD_initCStream_usingDict() :</a></li>
<li><a href="#Chapter22">! ZSTD_initCStream_advanced() :</a></li>
<li><a href="#Chapter23">! ZSTD_initCStream_usingCDict() :</a></li>
<li><a href="#Chapter24">! ZSTD_initCStream_usingCDict_advanced() :</a></li>
<li><a href="#Chapter25">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter26">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter27">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter28">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter29">Block level API</a></li>
<li><a href="#Chapter26">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter27">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter28">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter29">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter30">Block level API</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@ -72,8 +73,14 @@
<a name="Chapter2"></a><h2>Version</h2><pre></pre>
<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b>
</b></pre><BR>
<pre><b>unsigned ZSTD_versionNumber(void);
</b><p> Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE).
</p></pre><BR>
<pre><b>const char* ZSTD_versionString(void);
</b><p> Return runtime library version, like "1.4.5". Requires v1.3.0+.
</p></pre><BR>
<a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
<pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
@ -308,16 +315,20 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_dictIDFlag=202, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
</b>/* multi-threading parameters */<b>
</b>/* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).<b>
* They return an error otherwise. */
</b>/* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).<b>
* Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
* In a situation where it's unknown if the linked library supports multi-threading or not,
* setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
*/
ZSTD_c_nbWorkers=400, </b>/* Select how many threads will be spawned to compress in parallel.<b>
* When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
* When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
* while compression work is performed in parallel, within worker threads.
* while compression is performed in parallel, within worker thread(s).
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
* More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
* compression is performed inside Caller's thread, and all invocations are blocking */
ZSTD_c_jobSize=401, </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b>
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
@ -346,6 +357,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* ZSTD_c_literalCompressionMode
* ZSTD_c_targetCBlockSize
* ZSTD_c_srcSizeHint
* ZSTD_c_enableDedicatedDictSearch
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@ -356,7 +368,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004
ZSTD_c_experimentalParam7=1004,
ZSTD_c_experimentalParam8=1005
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>typedef struct {
@ -456,11 +469,13 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* At the time of this writing, they include :
* ZSTD_d_format
* ZSTD_d_stableOutBuffer
* ZSTD_d_forceIgnoreChecksum
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly
*/
ZSTD_d_experimentalParam1=1000,
ZSTD_d_experimentalParam2=1001
ZSTD_d_experimentalParam2=1001,
ZSTD_d_experimentalParam3=1002
} ZSTD_dParameter;
</b></pre><BR>
@ -591,8 +606,9 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
- Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
- output->pos must be <= dstCapacity, input->pos must be <= srcSize
- output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- endOp must be a valid directive
- When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
- When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
- When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
and then immediately returns, just indicating that there is some data remaining to be flushed.
The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@ -951,6 +967,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Decoder cannot recognise automatically this format, requiring this instruction. */
} ZSTD_format_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* Note: this enum controls ZSTD_d_forceIgnoreChecksum */<b>
ZSTD_d_validateChecksum = 0,
ZSTD_d_ignoreChecksum = 1
} ZSTD_forceIgnoreChecksum_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* Note: this enum and the behavior it controls are effectively internal<b>
* implementation details of the compressor. They are expected to continue
@ -1148,7 +1170,10 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</p></pre><BR>
<a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre>
<a name="Chapter17"></a><h2>This API is temporary and is expected to change or disappear in the future!</h2><pre>
<BR></pre>
<a name="Chapter18"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> Create a digested dictionary for compression
@ -1264,8 +1289,10 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
<pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
</b><p> Similar to ZSTD_CCtx_setParameter.
Set one compression parameter, selected by enum ZSTD_cParameter.
Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Parameters must be applied to a ZSTD_CCtx using
ZSTD_CCtx_setParametersUsingCCtxParams().
@result : a code representing success or failure (which can be tested with
ZSTD_isError()).
</p></pre><BR>
@ -1298,7 +1325,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR>
<a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre>
<a name="Chapter19"></a><h2>Advanced decompression functions</h2><pre></pre>
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
@ -1360,7 +1387,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR>
<a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API.
<a name="Chapter20"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API.
Once Advanced API reaches "stable" status,
redundant functions will be deprecated, and then at some point removed.
<BR></pre>
@ -1382,7 +1409,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel,
unsigned long long pledgedSrcSize);
</pre></b><BR>
<a name="Chapter20"></a><h2>! ZSTD_initCStream_usingDict() :</h2><pre> This function is deprecated, and is equivalent to:
<a name="Chapter21"></a><h2>! ZSTD_initCStream_usingDict() :</h2><pre> This function is deprecated, and is equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@ -1395,7 +1422,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
<BR></pre>
<a name="Chapter21"></a><h2>! ZSTD_initCStream_advanced() :</h2><pre> This function is deprecated, and is approximately equivalent to:
<a name="Chapter22"></a><h2>! ZSTD_initCStream_advanced() :</h2><pre> This function is deprecated, and is approximately equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd parameter and leave the rest as-is.
for ((param, value) : params) {
@ -1411,7 +1438,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
<BR></pre>
<a name="Chapter22"></a><h2>! ZSTD_initCStream_usingCDict() :</h2><pre> This function is deprecated, and equivalent to:
<a name="Chapter23"></a><h2>! ZSTD_initCStream_usingCDict() :</h2><pre> This function is deprecated, and equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_refCDict(zcs, cdict);
@ -1420,7 +1447,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
<BR></pre>
<a name="Chapter23"></a><h2>! ZSTD_initCStream_usingCDict_advanced() :</h2><pre> This function is DEPRECATED, and is approximately equivalent to:
<a name="Chapter24"></a><h2>! ZSTD_initCStream_usingCDict_advanced() :</h2><pre> This function is DEPRECATED, and is approximately equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd frame parameter and leave the rest as-is.
for ((fParam, value) : fParams) {
@ -1488,7 +1515,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
*/
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
</pre></b><BR>
<a name="Chapter24"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
<a name="Chapter25"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
ZSTD_DCtx_refDDict(zds, ddict);
@ -1497,7 +1524,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
<BR></pre>
<a name="Chapter25"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
<a name="Chapter26"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
re-use decompression parameters from previous init; saves dictionary loading
@ -1505,14 +1532,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
<BR></pre>
<a name="Chapter26"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
<a name="Chapter27"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience.
<BR></pre>
<a name="Chapter27"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
<a name="Chapter28"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@ -1548,7 +1575,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR>
<a name="Chapter28"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
<a name="Chapter29"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
@ -1644,7 +1671,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR>
<a name="Chapter29"></a><h2>Block level API</h2><pre></pre>
<a name="Chapter30"></a><h2>Block level API</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.

12
lib/compress/zstd_ldm.c
View File

@ -559,20 +559,22 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
* Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
*/
static void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
U32 currPos = rawSeqStore->posInSequence + nbBytes;
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
if (currPos >= currSeq.litLength + currSeq.matchLength) {
currPos -= currSeq.litLength + currSeq.matchLength;
rawSeqStore->pos++;
if (currPos == 0) {
rawSeqStore->posInSequence = 0;
}
} else {
rawSeqStore->posInSequence = currPos;
break;
}
}
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
rawSeqStore->posInSequence = 0;
}
assert(rawSeqStore->posInSequence <=
rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
}
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
@ -597,7 +599,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
if (rawSeqStore->pos >= rawSeqStore->size) {
/* If we're done with rawSeqStore, invalidate the one in matchState as well */
ms->ldmSeqStore.pos = rawSeqStore->size;
ms->ldmSeqStore.size = 0;
}
return lastLLSize;
}

18
lib/compress/zstd_opt.c
View File

@ -780,20 +780,22 @@ typedef struct {
* Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'.
*/
static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
U32 currPos = rawSeqStore->posInSequence + nbBytes;
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
if (currPos >= currSeq.litLength + currSeq.matchLength) {
currPos -= currSeq.litLength + currSeq.matchLength;
rawSeqStore->pos++;
if (currPos == 0) {
rawSeqStore->posInSequence = 0;
}
} else {
rawSeqStore->posInSequence = currPos;
break;
}
}
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
rawSeqStore->posInSequence = 0;
}
assert(rawSeqStore->posInSequence <=
rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
}
/* ZSTD_opt_getNextMatchAndUpdateSeqStore():
@ -854,7 +856,7 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu
* and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches'
*/
static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
ZSTD_optLdm_t* optLdm, U32 currPosInBlock) {
ZSTD_optLdm_t* optLdm, U32 currPosInBlock) {
U32 posDiff = currPosInBlock - optLdm->startPosInBlock;
/* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */
U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
@ -891,7 +893,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
* Wrapper function to update ldm seq store and call ldm functions as necessary.
*/
static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches,
U32 currPosInBlock, U32 remainingBytes) {
U32 currPosInBlock, U32 remainingBytes) {
if (optLdm->seqStore.size == 0 && optLdm->seqStore.pos >= optLdm->seqStore.size) {
return;
}
@ -983,7 +985,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 const ll0 = !litlen;
U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
(U32)(ip-istart), (U32)(iend - ip));
(U32)(ip-istart), (U32)(iend - ip));
if (!nbMatches) { ip++; continue; }
/* initialize opt[0] */
@ -1100,7 +1102,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 matchNb;
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
(U32)(inr-istart), (U32)(iend-inr));
(U32)(inr-istart), (U32)(iend-inr));
if (!nbMatches) {
DEBUGLOG(7, "rPos:%u : no match found", cur);