v1.4.3: Merge pull request #1730 from facebook/dev
This commit is contained in:
Felix Handte
GitHub
commit
a3d655d225
@ -123,6 +123,10 @@ matrix:
|
||||
- make ppcinstall
|
||||
- make ppcfuzz
|
||||
|
||||
- name: Trusty (Versions Compatibility Test)
|
||||
script:
|
||||
- make -C tests versionsTest
|
||||
|
||||
# check release number
|
||||
- name: Tag-Specific Test
|
||||
if: tag =~ ^v[0-9]\.[0-9]
|
||||
|
||||
@ -1,3 +1,9 @@
|
||||
v1.4.3
|
||||
bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
|
||||
bug: Fix Buffer Overflow in v0.3 Decompression by @felixhandte (#1722)
|
||||
build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
|
||||
misc: Add NULL pointer check in util.c by @leeyoung624 (#1706)
|
||||
|
||||
v1.4.2
|
||||
bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
|
||||
bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
||||
<title>zstd 1.4.2 Manual</title>
|
||||
<title>zstd 1.4.3 Manual</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>zstd 1.4.2 Manual</h1>
|
||||
<h1>zstd 1.4.3 Manual</h1>
|
||||
<hr>
|
||||
<a name="Contents"></a><h2>Contents</h2>
|
||||
<ol>
|
||||
@ -1557,7 +1557,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
<a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
|
||||
|
||||
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Compressing and decompressing require a context structure
|
||||
@ -1568,12 +1568,14 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
+ copyCCtx() and copyDCtx() can be used too
|
||||
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
||||
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
||||
+ For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst` !
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
|
||||
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
|
||||
===> In which case, nothing is produced into `dst` !
|
||||
+ User __must__ test for such outcome and deal directly with uncompressed data
|
||||
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
|
||||
Doing so would mess up with statistics history, leading to potential data corruption.
|
||||
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
|
||||
+ In case of multiple successive blocks, should some of them be uncompressed,
|
||||
decoder must be informed of their existence in order to follow proper history.
|
||||
Use ZSTD_insertBlock() for such a case.
|
||||
|
||||
@ -57,6 +57,8 @@ extern "C" {
|
||||
=========================================*/
|
||||
#if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
#elif defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
#endif
|
||||
|
||||
#define STREAM_ACCUMULATOR_MIN_32 25
|
||||
@ -163,6 +165,8 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
return 31 - __builtin_clz (val);
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
return 31 - __CLZ(val);
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
|
||||
11, 14, 16, 18, 22, 25, 3, 30,
|
||||
|
||||
@ -23,7 +23,7 @@
|
||||
# define INLINE_KEYWORD
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__)
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define FORCE_INLINE_ATTR __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define FORCE_INLINE_ATTR __forceinline
|
||||
@ -65,7 +65,7 @@
|
||||
#ifdef _MSC_VER
|
||||
# define FORCE_NOINLINE static __declspec(noinline)
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define FORCE_NOINLINE static __attribute__((__noinline__))
|
||||
# else
|
||||
# define FORCE_NOINLINE static
|
||||
@ -76,7 +76,7 @@
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
|
||||
#else
|
||||
# define TARGET_ATTRIBUTE(target)
|
||||
|
||||
@ -102,7 +102,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define MEM_FORCE_MEMORY_ACCESS 2
|
||||
# elif defined(__INTEL_COMPILER) || defined(__GNUC__)
|
||||
# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
@ -53,7 +53,8 @@
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define XXH_FORCE_MEMORY_ACCESS 2
|
||||
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \
|
||||
defined(__ICCARM__)
|
||||
# define XXH_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
@ -120,7 +121,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
|
||||
# define INLINE_KEYWORD
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__)
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define FORCE_INLINE_ATTR __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define FORCE_INLINE_ATTR __forceinline
|
||||
@ -206,7 +207,12 @@ static U64 XXH_read64(const void* memPtr)
|
||||
# define XXH_rotl32(x,r) _rotl(x,r)
|
||||
# define XXH_rotl64(x,r) _rotl64(x,r)
|
||||
#else
|
||||
#if defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
# define XXH_rotl32(x,r) __ROR(x,(32 - r))
|
||||
#else
|
||||
# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
#endif
|
||||
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
|
||||
#endif
|
||||
|
||||
|
||||
@ -56,9 +56,9 @@ extern "C" {
|
||||
/**
|
||||
* Return the specified error if the condition evaluates to true.
|
||||
*
|
||||
* In debug modes, prints additional information. In order to do that
|
||||
* (particularly, printing the conditional that failed), this can't just wrap
|
||||
* RETURN_ERROR().
|
||||
* In debug modes, prints additional information.
|
||||
* In order to do that (particularly, printing the conditional that failed),
|
||||
* this can't just wrap RETURN_ERROR().
|
||||
*/
|
||||
#define RETURN_ERROR_IF(cond, err, ...) \
|
||||
if (cond) { \
|
||||
@ -324,6 +324,8 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
return 31 - __CLZ(val);
|
||||
# else /* Software version */
|
||||
static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
|
||||
@ -1955,7 +1955,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
|
||||
size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
BYTE* seqHead;
|
||||
BYTE* lastNCount = NULL;
|
||||
|
||||
@ -1964,7 +1964,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
|
||||
/* Compress literals */
|
||||
{ const BYTE* const literals = seqStorePtr->litStart;
|
||||
size_t const litSize = seqStorePtr->lit - literals;
|
||||
size_t const litSize = (size_t)(seqStorePtr->lit - literals);
|
||||
size_t const cSize = ZSTD_compressLiterals(
|
||||
&prevEntropy->huf, &nextEntropy->huf,
|
||||
cctxParams->cParams.strategy,
|
||||
@ -1991,7 +1991,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
if (nbSeq==0) {
|
||||
/* Copy the old tables over as if we repeated them */
|
||||
memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
|
||||
return op - ostart;
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
/* seqHead : flags for FSE encoding type */
|
||||
@ -2012,7 +2012,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
ZSTD_defaultAllowed, strategy);
|
||||
assert(set_basic < set_compressed && set_rle < set_compressed);
|
||||
assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
||||
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
|
||||
{ size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
|
||||
count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
|
||||
prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable),
|
||||
workspace, wkspSize);
|
||||
@ -2035,7 +2035,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
OF_defaultNorm, OF_defaultNormLog,
|
||||
defaultPolicy, strategy);
|
||||
assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
||||
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
|
||||
{ size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
|
||||
count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
|
||||
prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable),
|
||||
workspace, wkspSize);
|
||||
@ -2056,7 +2056,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
ML_defaultNorm, ML_defaultNormLog,
|
||||
ZSTD_defaultAllowed, strategy);
|
||||
assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
||||
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
|
||||
{ size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
|
||||
count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
|
||||
prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable),
|
||||
workspace, wkspSize);
|
||||
@ -2070,7 +2070,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
|
||||
|
||||
{ size_t const bitstreamSize = ZSTD_encodeSequences(
|
||||
op, oend - op,
|
||||
op, (size_t)(oend - op),
|
||||
CTable_MatchLength, mlCodeTable,
|
||||
CTable_OffsetBits, ofCodeTable,
|
||||
CTable_LitLength, llCodeTable,
|
||||
@ -2097,7 +2097,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
||||
}
|
||||
|
||||
DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart));
|
||||
return op - ostart;
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t
|
||||
@ -2270,7 +2270,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
|
||||
{
|
||||
size_t cSize;
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
|
||||
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate);
|
||||
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
|
||||
(unsigned)zc->blockState.matchState.nextToUpdate);
|
||||
|
||||
{ const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
|
||||
FORWARD_IF_ERROR(bss);
|
||||
@ -2538,8 +2539,9 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
|
||||
|
||||
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
|
||||
RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong);
|
||||
DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
|
||||
{ size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
|
||||
RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); }
|
||||
|
||||
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
|
||||
}
|
||||
@ -2564,7 +2566,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
|
||||
if (srcSize <= HASH_READ_SIZE) return 0;
|
||||
|
||||
while (iend - ip > HASH_READ_SIZE) {
|
||||
size_t const remaining = iend - ip;
|
||||
size_t const remaining = (size_t)(iend - ip);
|
||||
size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX);
|
||||
const BYTE* const ichunk = ip + chunk;
|
||||
|
||||
|
||||
@ -134,9 +134,15 @@ typedef struct {
|
||||
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
|
||||
struct ZSTD_matchState_t {
|
||||
ZSTD_window_t window; /* State for window round buffer management */
|
||||
U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */
|
||||
U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
|
||||
* When loadedDictEnd != 0, a dictionary is in use, and still valid.
|
||||
* This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance.
|
||||
* Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity().
|
||||
* When dict referential is copied into active context (i.e. not attached),
|
||||
* loadedDictEnd == dictSize, since referential starts from zero.
|
||||
*/
|
||||
U32 nextToUpdate; /* index from which to continue table update */
|
||||
U32 hashLog3; /* dispatch table : larger == faster, more memory */
|
||||
U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
|
||||
U32* hashTable;
|
||||
U32* hashTable3;
|
||||
U32* chainTable;
|
||||
@ -350,7 +356,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
|
||||
/* copy Literals */
|
||||
assert(seqStorePtr->maxNbLit <= 128 KB);
|
||||
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap);
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap);
|
||||
seqStorePtr->lit += litLength;
|
||||
|
||||
/* literal Length */
|
||||
@ -763,24 +769,37 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
|
||||
|
||||
/* Similar to ZSTD_window_enforceMaxDist(),
|
||||
* but only invalidates dictionary
|
||||
* when input progresses beyond window size. */
|
||||
* when input progresses beyond window size.
|
||||
* assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL)
|
||||
* loadedDictEnd uses same referential as window->base
|
||||
* maxDist is the window size */
|
||||
MEM_STATIC void
|
||||
ZSTD_checkDictValidity(ZSTD_window_t* window,
|
||||
ZSTD_checkDictValidity(const ZSTD_window_t* window,
|
||||
const void* blockEnd,
|
||||
U32 maxDist,
|
||||
U32* loadedDictEndPtr,
|
||||
const ZSTD_matchState_t** dictMatchStatePtr)
|
||||
{
|
||||
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
|
||||
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
|
||||
assert(loadedDictEndPtr != NULL);
|
||||
assert(dictMatchStatePtr != NULL);
|
||||
{ U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
|
||||
U32 const loadedDictEnd = *loadedDictEndPtr;
|
||||
DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
|
||||
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
|
||||
assert(blockEndIdx >= loadedDictEnd);
|
||||
|
||||
if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) {
|
||||
/* On reaching window size, dictionaries are invalidated */
|
||||
if (loadedDictEndPtr) *loadedDictEndPtr = 0;
|
||||
if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
|
||||
}
|
||||
if (blockEndIdx > loadedDictEnd + maxDist) {
|
||||
/* On reaching window size, dictionaries are invalidated.
|
||||
* For simplification, if window size is reached anywhere within next block,
|
||||
* the dictionary is invalidated for the full block.
|
||||
*/
|
||||
DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
|
||||
*loadedDictEndPtr = 0;
|
||||
*dictMatchStatePtr = NULL;
|
||||
} else {
|
||||
if (*loadedDictEndPtr != 0) {
|
||||
DEBUGLOG(6, "dictionary considered valid for current block");
|
||||
} } }
|
||||
}
|
||||
|
||||
/**
|
||||
@ -822,6 +841,17 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
||||
return contiguous;
|
||||
}
|
||||
|
||||
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
|
||||
{
|
||||
U32 const maxDistance = 1U << windowLog;
|
||||
U32 const lowestValid = ms->window.lowLimit;
|
||||
U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
|
||||
U32 const isDictionary = (ms->loadedDictEnd != 0);
|
||||
U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
|
||||
return matchLowest;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* debug functions */
|
||||
#if (DEBUGLEVEL>=2)
|
||||
|
||||
@ -65,6 +65,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
|
||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
||||
const U32 lowestValid = ms->window.dictLimit;
|
||||
const U32 maxDistance = 1U << cParams->windowLog;
|
||||
/* presumes that, if there is a dictionary, it must be using Attach mode */
|
||||
const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid;
|
||||
const BYTE* const prefixLowest = base + prefixLowestIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
@ -369,9 +370,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
||||
const U32 maxDistance = 1U << cParams->windowLog;
|
||||
const U32 lowestValid = ms->window.lowLimit;
|
||||
const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid;
|
||||
const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
|
||||
const U32 dictStartIndex = lowLimit;
|
||||
const U32 dictLimit = ms->window.dictLimit;
|
||||
const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit;
|
||||
|
||||
@ -71,6 +71,7 @@ size_t ZSTD_compressBlock_fast_generic(
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
|
||||
ip0 += (ip0 == prefixStart);
|
||||
ip1 = ip0 + 1;
|
||||
{
|
||||
@ -239,6 +240,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic");
|
||||
ip += (dictAndPrefixLength == 0);
|
||||
/* dictMatchState repCode checks don't currently handle repCode == 0
|
||||
* disabling. */
|
||||
@ -379,9 +381,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
||||
const U32 maxDistance = 1U << cParams->windowLog;
|
||||
const U32 validLow = ms->window.lowLimit;
|
||||
const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow;
|
||||
const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
|
||||
const U32 dictStartIndex = lowLimit;
|
||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
||||
const U32 dictLimit = ms->window.dictLimit;
|
||||
@ -392,6 +392,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic");
|
||||
|
||||
/* switch to "regular" variant if extDict is invalidated due to maxDistance */
|
||||
if (prefixStartIndex == dictStartIndex)
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls);
|
||||
@ -412,8 +414,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
|
||||
} else {
|
||||
@ -423,8 +425,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
ip += ((ip-anchor) >> kSearchStrength) + stepSize;
|
||||
continue;
|
||||
}
|
||||
{ const BYTE* matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
{ const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
U32 offset;
|
||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
|
||||
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
@ -451,7 +453,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
|
||||
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
|
||||
ip += repLength2;
|
||||
|
||||
@ -242,9 +242,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
||||
|
||||
const BYTE* const base = ms->window.base;
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const maxDistance = 1U << cParams->windowLog;
|
||||
U32 const windowValid = ms->window.lowLimit;
|
||||
U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid;
|
||||
U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog);
|
||||
|
||||
U32* const bt = ms->chainTable;
|
||||
U32 const btLog = cParams->chainLog - 1;
|
||||
@ -497,8 +495,10 @@ size_t ZSTD_HcFindBestMatch_generic (
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
const U32 current = (U32)(ip-base);
|
||||
const U32 maxDistance = 1U << cParams->windowLog;
|
||||
const U32 lowValid = ms->window.lowLimit;
|
||||
const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid;
|
||||
const U32 lowestValid = ms->window.lowLimit;
|
||||
const U32 withinMaxDistance = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
|
||||
const U32 isDictionary = (ms->loadedDictEnd != 0);
|
||||
const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
|
||||
const U32 minChain = current > chainSize ? current - chainSize : 0;
|
||||
U32 nbAttempts = 1U << cParams->searchLog;
|
||||
size_t ml=4-1;
|
||||
@ -619,12 +619,14 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
|
||||
/* *******************************
|
||||
* Common parser - lazy strategy
|
||||
*********************************/
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t ZSTD_compressBlock_lazy_generic(
|
||||
typedef enum { search_hashChain, search_binaryTree } searchMethod_e;
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
ZSTD_compressBlock_lazy_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore,
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize,
|
||||
const U32 searchMethod, const U32 depth,
|
||||
const searchMethod_e searchMethod, const U32 depth,
|
||||
ZSTD_dictMode_e const dictMode)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
@ -640,8 +642,10 @@ size_t ZSTD_compressBlock_lazy_generic(
|
||||
ZSTD_matchState_t* ms,
|
||||
const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
|
||||
searchMax_f const searchMax = dictMode == ZSTD_dictMatchState ?
|
||||
(searchMethod ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) :
|
||||
(searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS);
|
||||
(searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS
|
||||
: ZSTD_HcFindBestMatch_dictMatchState_selectMLS) :
|
||||
(searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_selectMLS
|
||||
: ZSTD_HcFindBestMatch_selectMLS);
|
||||
U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0;
|
||||
|
||||
const ZSTD_matchState_t* const dms = ms->dictMatchState;
|
||||
@ -850,7 +854,7 @@ _storeSequence:
|
||||
rep[1] = offset_2 ? offset_2 : savedOffset;
|
||||
|
||||
/* Return the last literals size */
|
||||
return iend - anchor;
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
|
||||
|
||||
@ -858,56 +862,56 @@ size_t ZSTD_compressBlock_btlazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_noDict);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_lazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_noDict);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_lazy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_noDict);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_greedy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_noDict);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_dictMatchState);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_dictMatchState);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_dictMatchState);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_greedy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_dictMatchState);
|
||||
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState);
|
||||
}
|
||||
|
||||
|
||||
@ -916,7 +920,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore,
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize,
|
||||
const U32 searchMethod, const U32 depth)
|
||||
const searchMethod_e searchMethod, const U32 depth)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
@ -934,7 +938,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
||||
typedef size_t (*searchMax_f)(
|
||||
ZSTD_matchState_t* ms,
|
||||
const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
|
||||
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS;
|
||||
searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS;
|
||||
|
||||
U32 offset_1 = rep[0], offset_2 = rep[1];
|
||||
|
||||
@ -1075,7 +1079,7 @@ _storeSequence:
|
||||
rep[1] = offset_2;
|
||||
|
||||
/* Return the last literals size */
|
||||
return iend - anchor;
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
|
||||
|
||||
@ -1083,7 +1087,7 @@ size_t ZSTD_compressBlock_greedy_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 0);
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_lazy_extDict(
|
||||
@ -1091,7 +1095,7 @@ size_t ZSTD_compressBlock_lazy_extDict(
|
||||
void const* src, size_t srcSize)
|
||||
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 1);
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_lazy2_extDict(
|
||||
@ -1099,7 +1103,7 @@ size_t ZSTD_compressBlock_lazy2_extDict(
|
||||
void const* src, size_t srcSize)
|
||||
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 2);
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2_extDict(
|
||||
@ -1107,5 +1111,5 @@ size_t ZSTD_compressBlock_btlazy2_extDict(
|
||||
void const* src, size_t srcSize)
|
||||
|
||||
{
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 1, 2);
|
||||
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2);
|
||||
}
|
||||
|
||||
@ -552,7 +552,6 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
|
||||
U32 const maxDistance = 1U << cParams->windowLog;
|
||||
const BYTE* const base = ms->window.base;
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const hashLog = cParams->hashLog;
|
||||
@ -569,8 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
const BYTE* const prefixStart = base + dictLimit;
|
||||
U32 const btLow = (btMask >= current) ? 0 : current - btMask;
|
||||
U32 const windowValid = ms->window.lowLimit;
|
||||
U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid;
|
||||
U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog);
|
||||
U32 const matchLow = windowLow ? windowLow : 1;
|
||||
U32* smallerPtr = bt + 2*(current&btMask);
|
||||
U32* largerPtr = bt + 2*(current&btMask) + 1;
|
||||
@ -674,19 +672,21 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
||||
|
||||
while (nbCompares-- && (matchIndex >= matchLow)) {
|
||||
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
const BYTE* match;
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
assert(current > matchIndex);
|
||||
|
||||
if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) {
|
||||
assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */
|
||||
match = base + matchIndex;
|
||||
if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
|
||||
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit);
|
||||
} else {
|
||||
match = dictBase + matchIndex;
|
||||
assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
|
||||
if (matchIndex+matchLength >= dictLimit)
|
||||
match = base + matchIndex; /* prepare for match[matchLength] */
|
||||
match = base + matchIndex; /* prepare for match[matchLength] read */
|
||||
}
|
||||
|
||||
if (matchLength > bestLength) {
|
||||
|
||||
@ -574,9 +574,10 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
|
||||
}
|
||||
|
||||
/** ZSTD_insertBlock() :
|
||||
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
|
||||
* insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
|
||||
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
|
||||
ZSTD_checkContinuity(dctx, blockStart);
|
||||
dctx->previousDstEnd = (const char*)blockStart + blockSize;
|
||||
return blockSize;
|
||||
|
||||
@ -79,6 +79,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
|
||||
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected);
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) src;
|
||||
@ -87,6 +88,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
switch(litEncType)
|
||||
{
|
||||
case set_repeat:
|
||||
DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
|
||||
RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted);
|
||||
/* fall-through */
|
||||
|
||||
@ -116,7 +118,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
/* 2 - 2 - 18 - 18 */
|
||||
lhSize = 5;
|
||||
litSize = (lhc >> 4) & 0x3FFFF;
|
||||
litCSize = (lhc >> 22) + (istart[4] << 10);
|
||||
litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
|
||||
break;
|
||||
}
|
||||
RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected);
|
||||
@ -391,7 +393,8 @@ ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
assert(normalizedCounter[s]>=0);
|
||||
symbolNext[s] = (U16)normalizedCounter[s];
|
||||
} } }
|
||||
memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
}
|
||||
|
||||
@ -2530,6 +2530,7 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
||||
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
||||
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
||||
{
|
||||
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
||||
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
||||
memcpy(dctx->litBuffer, istart, litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
|
||||
18
lib/zstd.h
18
lib/zstd.h
@ -71,7 +71,7 @@ extern "C" {
|
||||
/*------ Version ------*/
|
||||
#define ZSTD_VERSION_MAJOR 1
|
||||
#define ZSTD_VERSION_MINOR 4
|
||||
#define ZSTD_VERSION_RELEASE 2
|
||||
#define ZSTD_VERSION_RELEASE 3
|
||||
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
|
||||
@ -1909,7 +1909,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
/*!
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Compressing and decompressing require a context structure
|
||||
@ -1920,12 +1920,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
+ copyCCtx() and copyDCtx() can be used too
|
||||
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
||||
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
||||
+ For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst` !
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
|
||||
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
|
||||
===> In which case, nothing is produced into `dst` !
|
||||
+ User __must__ test for such outcome and deal directly with uncompressed data
|
||||
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
|
||||
Doing so would mess up with statistics history, leading to potential data corruption.
|
||||
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
|
||||
+ In case of multiple successive blocks, should some of them be uncompressed,
|
||||
decoder must be informed of their existence in order to follow proper history.
|
||||
Use ZSTD_insertBlock() for such a case.
|
||||
|
||||
@ -55,17 +55,18 @@ static U32 RDG_rand(U32* src)
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
typedef U32 fixedPoint_24_8;
|
||||
|
||||
static void RDG_fillLiteralDistrib(BYTE* ldt, double ld)
|
||||
static void RDG_fillLiteralDistrib(BYTE* ldt, fixedPoint_24_8 ld)
|
||||
{
|
||||
BYTE const firstChar = (ld<=0.0) ? 0 : '(';
|
||||
BYTE const lastChar = (ld<=0.0) ? 255 : '}';
|
||||
BYTE character = (ld<=0.0) ? 0 : '0';
|
||||
U32 u;
|
||||
|
||||
if (ld<=0.0) ld = 0.0;
|
||||
if (ld<=0) ld = 0;
|
||||
for (u=0; u<LTSIZE; ) {
|
||||
U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
|
||||
U32 const weight = (((LTSIZE - u) * ld) >> 8) + 1;
|
||||
U32 const end = MIN ( u + weight , LTSIZE);
|
||||
while (u < end) ldt[u++] = character;
|
||||
character++;
|
||||
@ -92,7 +93,8 @@ static U32 RDG_randLength(U32* seedPtr)
|
||||
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
|
||||
}
|
||||
|
||||
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, U32* seedPtr)
|
||||
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize,
|
||||
double matchProba, const BYTE* ldt, U32* seedPtr)
|
||||
{
|
||||
BYTE* const buffPtr = (BYTE*)buffer;
|
||||
U32 const matchProba32 = (U32)(32768 * matchProba);
|
||||
@ -128,13 +130,13 @@ static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, doubl
|
||||
U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
|
||||
U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
|
||||
size_t match = pos - offset;
|
||||
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
|
||||
while (pos < d) { buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ }
|
||||
prevOffset = offset;
|
||||
} else {
|
||||
/* Literal (noise) */
|
||||
U32 const length = RDG_randLength(seedPtr);
|
||||
U32 const d = (U32) MIN(pos + length, buffSize);
|
||||
while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt);
|
||||
while (pos < d) { buffPtr[pos++] = RDG_genChar(seedPtr, ldt); }
|
||||
} }
|
||||
}
|
||||
|
||||
@ -145,7 +147,7 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba
|
||||
BYTE ldt[LTSIZE];
|
||||
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
|
||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
||||
RDG_fillLiteralDistrib(ldt, litProba);
|
||||
RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001));
|
||||
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32);
|
||||
}
|
||||
|
||||
@ -163,7 +165,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
|
||||
if (buff==NULL) { perror("datagen"); exit(1); }
|
||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
||||
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
|
||||
RDG_fillLiteralDistrib(ldt, litProba);
|
||||
RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001));
|
||||
SET_BINARY_MODE(stdout);
|
||||
|
||||
/* Generate initial dict */
|
||||
|
||||
@ -494,7 +494,7 @@ int UTIL_countPhysicalCores(void)
|
||||
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
|
||||
if (strncmp(buff, "siblings", 8) == 0) {
|
||||
const char* const sep = strchr(buff, ':');
|
||||
if (*sep == '\0') {
|
||||
if (sep == NULL || *sep == '\0') {
|
||||
/* formatting was broken? */
|
||||
goto failed;
|
||||
}
|
||||
@ -503,7 +503,7 @@ int UTIL_countPhysicalCores(void)
|
||||
}
|
||||
if (strncmp(buff, "cpu cores", 9) == 0) {
|
||||
const char* const sep = strchr(buff, ':');
|
||||
if (*sep == '\0') {
|
||||
if (sep == NULL || *sep == '\0') {
|
||||
/* formatting was broken? */
|
||||
goto failed;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "July 2019" "zstd 1.4.2" "User Commands"
|
||||
.TH "ZSTD" "1" "August 2019" "zstd 1.4.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTDGREP" "1" "July 2019" "zstd 1.4.2" "User Commands"
|
||||
.TH "ZSTDGREP" "1" "August 2019" "zstd 1.4.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTDLESS" "1" "July 2019" "zstd 1.4.2" "User Commands"
|
||||
.TH "ZSTDLESS" "1" "August 2019" "zstd 1.4.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdless\fR \- view zstandard\-compressed files
|
||||
|
||||
177
tests/fuzzer.c
177
tests/fuzzer.c
@ -123,9 +123,10 @@ static U32 FUZ_highbit32(U32 v32)
|
||||
exit(1); \
|
||||
} }
|
||||
|
||||
#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
|
||||
#define CHECK(fn) { CHECK_V(err, fn); }
|
||||
#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
|
||||
#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
|
||||
#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
|
||||
#define CHECK(fn) { CHECK_NEWV(err, fn); }
|
||||
#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
|
||||
|
||||
#define CHECK_OP(op, lhs, rhs) { \
|
||||
if (!((lhs) op (rhs))) { \
|
||||
@ -231,7 +232,7 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
|
||||
|
||||
/* advanced MT API test */
|
||||
if (part <= 3)
|
||||
{ unsigned nbThreads;
|
||||
{ int nbThreads;
|
||||
for (nbThreads=1; nbThreads<=4; nbThreads++) {
|
||||
int compressionLevel;
|
||||
for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
|
||||
@ -242,7 +243,7 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
|
||||
CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
|
||||
ZSTD_freeCCtx(cctx);
|
||||
DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
|
||||
DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
|
||||
nbThreads, compressionLevel);
|
||||
FUZ_displayMallocStats(malcount);
|
||||
} } }
|
||||
@ -307,7 +308,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||
* Unit tests
|
||||
=============================================*/
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
static int basicUnitTests(U32 const seed, double compressibility)
|
||||
{
|
||||
size_t const CNBuffSize = 5 MB;
|
||||
void* const CNBuffer = malloc(CNBuffSize);
|
||||
@ -345,10 +346,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
if (cctx==NULL) goto _output_error;
|
||||
CHECKPLUS(r, ZSTD_compressCCtx(cctx,
|
||||
CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
CNBuffer, CNBuffSize, 1),
|
||||
cSize=r );
|
||||
CNBuffer, CNBuffSize, 1) );
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
|
||||
@ -450,12 +450,11 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
|
||||
{ /* create compressed buffer with content size missing */
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
|
||||
CHECKPLUS(r, ZSTD_compress2(cctx,
|
||||
CHECK_VAR(cSize, ZSTD_compress2(cctx,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
CNBuffer, CNBuffSize),
|
||||
cSize=r );
|
||||
CNBuffer, CNBuffSize) );
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
{ /* ensure frame content size is missing */
|
||||
@ -742,10 +741,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : simple compression test with static CCtx : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx,
|
||||
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
|
||||
cSize=r );
|
||||
CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
|
||||
(unsigned)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
@ -777,7 +775,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
CHECK( ZSTD_initCStream(staticCCtx, 1) );
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : init CStream with dictionary (should fail) : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
|
||||
{ size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
|
||||
if (!ZSTD_isError(r)) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
@ -810,20 +808,19 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
/* ZSTDMT simple MT compression test */
|
||||
DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
|
||||
{ ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
|
||||
{ ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(2);
|
||||
if (mtctx==NULL) {
|
||||
DISPLAY("mtctx : mot enough memory, aborting \n");
|
||||
DISPLAY("mtctx : not enough memory, aborting \n");
|
||||
testResult = 1;
|
||||
goto _end;
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
|
||||
CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx,
|
||||
CHECK_VAR(cSize, ZSTDMT_compressCCtx(mtctx,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
CNBuffer, CNBuffSize,
|
||||
1),
|
||||
cSize=r );
|
||||
1) );
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
|
||||
@ -850,11 +847,10 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
{ ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, 0);
|
||||
params.fParams.checksumFlag = 1;
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
CHECKPLUS(r, ZSTDMT_compress_advanced(mtctx,
|
||||
CHECK_VAR(cSize, ZSTDMT_compress_advanced(mtctx,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
CNBuffer, CNBuffSize,
|
||||
NULL, params, 3 /*overlapRLog*/),
|
||||
cSize=r );
|
||||
NULL, params, 3 /*overlapRLog*/) );
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
@ -928,10 +924,10 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
/* only use the first half so we don't push against size limit of compressedBuffer */
|
||||
size_t const segSize = (CNBuffSize / 2) / segs;
|
||||
for (i = 0; i < segs; i++) {
|
||||
CHECK_V(r, ZSTD_compress(
|
||||
(BYTE *)compressedBuffer + off, CNBuffSize - off,
|
||||
(BYTE *)CNBuffer + segSize * i,
|
||||
segSize, 5));
|
||||
CHECK_NEWV(r, ZSTD_compress(
|
||||
(BYTE*)compressedBuffer + off, CNBuffSize - off,
|
||||
(BYTE*)CNBuffer + segSize * (size_t)i, segSize,
|
||||
5) );
|
||||
off += r;
|
||||
if (i == segs/2) {
|
||||
/* insert skippable frame */
|
||||
@ -956,7 +952,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
|
||||
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
|
||||
{ CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
|
||||
if (r != CNBuffSize / 2) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
@ -983,7 +979,8 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
|
||||
cSize = 0;
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, compressedBufferSize,
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize += r);
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
|
||||
@ -999,7 +996,8 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
|
||||
{ size_t const cSizeOrig = cSize;
|
||||
cSize = 0;
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, compressedBufferSize,
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize += r);
|
||||
if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
|
||||
@ -1024,7 +1022,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
|
||||
{ size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
|
||||
void* ddictBuffer = malloc(ddictBufferSize);
|
||||
void* const ddictBuffer = malloc(ddictBufferSize);
|
||||
if (ddictBuffer == NULL) goto _output_error;
|
||||
{ const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
|
||||
size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
|
||||
@ -1042,15 +1040,66 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
}
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
|
||||
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
|
||||
(const char*)CNBuffer + dictSize, testSize),
|
||||
cSize = r);
|
||||
CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
|
||||
(const char*)CNBuffer + dictSize, testSize) );
|
||||
{ ZSTD_frameHeader zfh;
|
||||
if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
|
||||
if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
|
||||
} }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
|
||||
size_t const flatdictSize = 22 KB;
|
||||
size_t const contentSize = 9 KB;
|
||||
const void* const dict = (const char*)CNBuffer;
|
||||
const void* const contentStart = (const char*)dict + flatdictSize;
|
||||
size_t const target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
|
||||
3770, 3770, 3770, 3750, 3750,
|
||||
3740, 3670, 3670, 3660, 3660,
|
||||
3660, 3660, 3660, 3660, 3660,
|
||||
3660, 3660, 3660 };
|
||||
size_t const target_wdict_cSize[22+1] = { 2830, 2890, 2890, 2820, 2940,
|
||||
2950, 2950, 2920, 2900, 2890,
|
||||
2910, 2910, 2910, 2770, 2760,
|
||||
2750, 2750, 2750, 2750, 2750,
|
||||
2750, 2750, 2750 };
|
||||
int l = 1;
|
||||
int const maxLevel = ZSTD_maxCLevel();
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
|
||||
assert(maxLevel == 22);
|
||||
RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
|
||||
DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", XXH64(contentStart, contentSize, 0), XXH64(dict, flatdictSize, 0));
|
||||
|
||||
for ( ; l <= maxLevel; l++) {
|
||||
size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
|
||||
contentStart, contentSize, l);
|
||||
if (nodict_cSize > target_nodict_cSize[l]) {
|
||||
DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
|
||||
l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
|
||||
goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
|
||||
l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
|
||||
}
|
||||
for ( l=1 ; l <= maxLevel; l++) {
|
||||
size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
contentStart, contentSize,
|
||||
dict, flatdictSize,
|
||||
l);
|
||||
if (wdict_cSize > target_wdict_cSize[l]) {
|
||||
DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
|
||||
l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
|
||||
goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
|
||||
l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "compression efficiency tests OK \n");
|
||||
}
|
||||
|
||||
ZSTD_freeCCtx(ctxOrig);
|
||||
ZSTD_freeCCtx(ctxDuplicated);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
@ -1598,6 +1647,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
size_t const sampleUnitSize = 8 KB;
|
||||
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
|
||||
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
|
||||
U32 seed32 = seed;
|
||||
ZDICT_cover_params_t params;
|
||||
U32 dictID;
|
||||
|
||||
@ -1610,8 +1660,8 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
|
||||
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.d = 1 + (FUZ_rand(&seed) % 16);
|
||||
params.k = params.d + (FUZ_rand(&seed) % 256);
|
||||
params.d = 1 + (FUZ_rand(&seed32) % 16);
|
||||
params.k = params.d + (FUZ_rand(&seed32) % 256);
|
||||
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
|
||||
CNBuffer, samplesSizes, nbSamples,
|
||||
params);
|
||||
@ -1830,13 +1880,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin(cctx, 5) );
|
||||
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
|
||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
|
||||
CHECK( ZSTD_decompressBegin(dctx) );
|
||||
{ CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
{ CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
@ -1851,30 +1900,36 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
assert(blockCSize != 0);
|
||||
if (ZSTD_isError(blockCSize)) goto _output_error;
|
||||
compressed += blockCSize;
|
||||
}
|
||||
}
|
||||
} }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
/* dictionary block compression */
|
||||
DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
|
||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize);
|
||||
if (ZSTD_isError(cSize2)) goto _output_error;
|
||||
memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* fake non-compressed block */
|
||||
cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
|
||||
(char*)CNBuffer+dictSize+2*blockSize, blockSize);
|
||||
if (ZSTD_isError(cSize2)) goto _output_error;
|
||||
CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
|
||||
RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
|
||||
{ CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
|
||||
assert(r == 0); /* non-compressible block */ }
|
||||
memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
|
||||
CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
|
||||
(char*)CNBuffer+dictSize+2*blockSize, blockSize));
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
|
||||
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
|
||||
{ CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
{ CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) {
|
||||
DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
|
||||
goto _output_error;
|
||||
} }
|
||||
memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
|
||||
ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
|
||||
{ CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
{ CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
|
||||
if (r != blockSize) {
|
||||
DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
|
||||
goto _output_error;
|
||||
} }
|
||||
assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
|
||||
@ -1900,7 +1955,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
sampleSize += 96 KB;
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
{ CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
||||
{ CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
||||
if (regenSize!=sampleSize) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
}
|
||||
@ -1909,12 +1964,11 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
#define ZEROESLENGTH 100
|
||||
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||
memset(CNBuffer, 0, ZEROESLENGTH);
|
||||
{ CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
|
||||
cSize = r; }
|
||||
CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
|
||||
{ CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
|
||||
if (r != ZEROESLENGTH) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
@ -1962,13 +2016,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
|
||||
{ CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
|
||||
CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
|
||||
CNBuffer, _3BYTESTESTLENGTH, 19) );
|
||||
cSize = r; }
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
|
||||
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
|
||||
{ CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
|
||||
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user