Merge from Cyan4973/dev060
This commit is contained in:
commit
19bd48f516
5
Makefile
5
Makefile
@ -89,9 +89,8 @@ gpptest: clean
|
||||
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
|
||||
armtest: clean
|
||||
# $(MAKE) -C $(ZSTDDIR) all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS=-static # MOREFLAGS="-Werror -static"
|
||||
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
# for Travis CI
|
||||
arminstall: clean
|
||||
@ -105,7 +104,7 @@ armtest-w-install: clean arminstall armtest
|
||||
|
||||
ppctest: clean
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS=-static # MOREFLAGS="-Werror -static"
|
||||
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
# for Travis CI
|
||||
ppcinstall: clean
|
||||
|
@ -53,6 +53,14 @@ extern "C" {
|
||||
#include "error_private.h" /* error codes and messages */
|
||||
|
||||
|
||||
/*=========================================
|
||||
* Target specific
|
||||
=========================================*/
|
||||
#if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
#endif
|
||||
|
||||
|
||||
/*-******************************************
|
||||
* bitStream encoding API (write forward)
|
||||
********************************************/
|
||||
@ -122,7 +130,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
|
||||
*/
|
||||
|
||||
|
||||
@ -141,7 +149,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Helper functions
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
{
|
||||
@ -165,6 +173,9 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
# endif
|
||||
}
|
||||
|
||||
/*===== Local Constants =====*/
|
||||
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* bitStream encoding
|
||||
@ -189,8 +200,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t ds
|
||||
Does not check for register overflow ! */
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
|
||||
{
|
||||
static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
|
||||
bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos;
|
||||
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
|
||||
bitC->bitPos += nbBits;
|
||||
}
|
||||
|
||||
@ -206,7 +216,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBi
|
||||
* unsafe version; does not check buffer overflow */
|
||||
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
|
||||
{
|
||||
size_t nbBytes = bitC->bitPos >> 3;
|
||||
size_t const nbBytes = bitC->bitPos >> 3;
|
||||
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
|
||||
bitC->ptr += nbBytes;
|
||||
bitC->bitPos &= 7;
|
||||
@ -218,7 +228,7 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
|
||||
* note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
|
||||
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
|
||||
{
|
||||
size_t nbBytes = bitC->bitPos >> 3;
|
||||
size_t const nbBytes = bitC->bitPos >> 3;
|
||||
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
|
||||
bitC->ptr += nbBytes;
|
||||
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
|
||||
@ -246,8 +256,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
|
||||
/*! BIT_initDStream() :
|
||||
* Initialize a BIT_DStream_t.
|
||||
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
|
||||
* `srcBuffer` must point at the beginning of a bitStream.
|
||||
* `srcSize` must be the exact size of the bitStream, in bytes.
|
||||
* `srcSize` must be the *exact* size of the bitStream, in bytes.
|
||||
* @return : size of stream (== srcSize) or an errorCode if a problem is detected
|
||||
*/
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
@ -255,15 +264,13 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
if (srcSize >= sizeof(size_t)) { /* normal case */
|
||||
U32 contain32;
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(contain32);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); }
|
||||
} else {
|
||||
U32 contain32;
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = bitD->start;
|
||||
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
||||
@ -277,15 +284,34 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
||||
default:;
|
||||
}
|
||||
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(contain32);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); }
|
||||
bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8;
|
||||
}
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getUpperBits(size_t bitD, U32 const start)
|
||||
{
|
||||
return bitD >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitD, U32 const nbBits, U32 const start)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
return __builtin_ia32_bextr_u64(bitD, (nbBits<<8) | start );
|
||||
#else
|
||||
return (bitD >> start) & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getLowerBits(size_t bitD, U32 const nbBits)
|
||||
{
|
||||
return bitD & BIT_mask[nbBits];
|
||||
}
|
||||
|
||||
/*! BIT_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified (bits are still present for next read/look).
|
||||
@ -293,15 +319,19 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
return __builtin_ia32_bextr_u64(bitD->bitContainer, (nbBits<<8) | (64 - bitD->bitsConsumed - nbBits) );
|
||||
#else
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BIT_lookBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
|
||||
@ -313,13 +343,13 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
}
|
||||
|
||||
/*! BIT_readBits() :
|
||||
* Read next n bits from local register.
|
||||
* pay attention to not read more than nbBits contained into local register.
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBits(bitD, nbBits);
|
||||
size_t const value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
@ -328,7 +358,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
||||
size_t const value = BIT_lookBitsFast(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
@ -95,6 +95,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
|
||||
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
|
||||
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
|
||||
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
|
||||
case PREFIX(init_missing): return "Context should be init first";
|
||||
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
||||
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
|
||||
|
@ -47,6 +47,7 @@ typedef enum {
|
||||
ZSTD_error_prefix_unknown,
|
||||
ZSTD_error_frameParameter_unsupported,
|
||||
ZSTD_error_frameParameter_unsupportedBy32bits,
|
||||
ZSTD_error_compressionParameter_unsupported,
|
||||
ZSTD_error_init_missing,
|
||||
ZSTD_error_memory_allocation,
|
||||
ZSTD_error_stage_wrong,
|
||||
|
162
lib/fse.c
162
lib/fse.c
@ -145,21 +145,18 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3)
|
||||
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
const unsigned tableSize = 1 << tableLog;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 const tableMask = tableSize - 1;
|
||||
void* const ptr = ct;
|
||||
U16* const tableU16 = ( (U16*) ptr) + 2;
|
||||
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
|
||||
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
||||
const unsigned step = FSE_tableStep(tableSize);
|
||||
unsigned cumul[FSE_MAX_SYMBOL_VALUE+2];
|
||||
U32 position = 0;
|
||||
U32 const step = FSE_tableStep(tableSize);
|
||||
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
|
||||
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
|
||||
U32 highThreshold = tableSize-1;
|
||||
unsigned symbol;
|
||||
unsigned i;
|
||||
|
||||
/* header */
|
||||
/* CTable header */
|
||||
tableU16[-2] = (U16) tableLog;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
|
||||
@ -167,42 +164,44 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||
|
||||
/* symbol start positions */
|
||||
cumul[0] = 0;
|
||||
for (i=1; i<=maxSymbolValue+1; i++) {
|
||||
if (normalizedCounter[i-1]==-1) { /* Low proba symbol */
|
||||
cumul[i] = cumul[i-1] + 1;
|
||||
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1);
|
||||
} else {
|
||||
cumul[i] = cumul[i-1] + normalizedCounter[i-1];
|
||||
} }
|
||||
cumul[maxSymbolValue+1] = tableSize+1;
|
||||
|
||||
/* Spread symbols */
|
||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||
int nbOccurences;
|
||||
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
|
||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
|
||||
} }
|
||||
|
||||
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
|
||||
|
||||
/* Build table */
|
||||
for (i=0; i<tableSize; i++) {
|
||||
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* note : static analyzer may not understand tableSymbol is properly initialized */
|
||||
tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
|
||||
{ U32 u;
|
||||
cumul[0] = 0;
|
||||
for (u=1; u<=maxSymbolValue+1; u++) {
|
||||
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
|
||||
cumul[u] = cumul[u-1] + 1;
|
||||
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
|
||||
} else {
|
||||
cumul[u] = cumul[u-1] + normalizedCounter[u-1];
|
||||
} }
|
||||
cumul[maxSymbolValue+1] = tableSize+1;
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 position = 0;
|
||||
U32 symbol;
|
||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||
int nbOccurences;
|
||||
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
|
||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
|
||||
} }
|
||||
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
|
||||
}
|
||||
|
||||
/* Build table */
|
||||
{ U32 u; for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
|
||||
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
|
||||
}}
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{
|
||||
{ unsigned total = 0;
|
||||
unsigned s;
|
||||
unsigned total = 0;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
switch (normalizedCounter[s])
|
||||
{
|
||||
case 0:
|
||||
break;
|
||||
case 0: break;
|
||||
case -1:
|
||||
case 1:
|
||||
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
|
||||
@ -211,8 +210,8 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
break;
|
||||
default :
|
||||
{
|
||||
U32 maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
|
||||
U32 minStatePlus = normalizedCounter[s] << maxBitsOut;
|
||||
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
|
||||
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
|
||||
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
|
||||
symbolTT[s].deltaFindState = total - normalizedCounter[s];
|
||||
total += normalizedCounter[s];
|
||||
@ -242,9 +241,8 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
|
||||
const U32 tableMask = tableSize-1;
|
||||
const U32 step = FSE_tableStep(tableSize);
|
||||
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
|
||||
U32 position = 0;
|
||||
U32 highThreshold = tableSize-1;
|
||||
const S16 largeLimit= (S16)(1 << (tableLog-1));
|
||||
S16 const largeLimit= (S16)(1 << (tableLog-1));
|
||||
U32 noLarge = 1;
|
||||
U32 s;
|
||||
|
||||
@ -264,24 +262,24 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
|
||||
} }
|
||||
|
||||
/* Spread symbols */
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
|
||||
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
||||
{ U32 position = 0;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
||||
}
|
||||
|
||||
/* Build Decoding table */
|
||||
{
|
||||
U32 i;
|
||||
for (i=0; i<tableSize; i++) {
|
||||
FSE_FUNCTION_TYPE symbol = (FSE_FUNCTION_TYPE)(tableDecode[i].symbol);
|
||||
{ U32 u;
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U16 nextState = symbolNext[symbol]++;
|
||||
tableDecode[i].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
|
||||
tableDecode[i].newState = (U16) ( (nextState << tableDecode[i].nbBits) - tableSize);
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
DTableH.fastMode = (U16)noLarge;
|
||||
@ -365,8 +363,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
bitStream >>= 16;
|
||||
bitCount -= 16;
|
||||
} }
|
||||
{
|
||||
short count = normalizedCounter[charnum++];
|
||||
{ short count = normalizedCounter[charnum++];
|
||||
const short max = (short)((2*threshold-1)-remaining);
|
||||
remaining -= FSE_abs(count);
|
||||
if (remaining<1) return ERROR(GENERIC);
|
||||
@ -465,8 +462,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
||||
else
|
||||
bitStream >>= 2;
|
||||
}
|
||||
{
|
||||
short const max = (short)((2*threshold-1)-remaining);
|
||||
{ short const max = (short)((2*threshold-1)-remaining);
|
||||
short count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
@ -509,11 +505,11 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
||||
* Counting histogram
|
||||
****************************************************************/
|
||||
/*! FSE_count_simple
|
||||
This function just counts byte values within @src,
|
||||
and store the histogram into @count.
|
||||
This function is unsafe : it doesn't check that all values within @src can fit into @count.
|
||||
For this reason, prefer using a table @count with 256 elements.
|
||||
@return : highest count for a single element
|
||||
This function just counts byte values within `src`,
|
||||
and store the histogram into table `count`.
|
||||
This function is unsafe : it doesn't check that all values within `src` can fit into `count`.
|
||||
For this reason, prefer using a table `count` with 256 elements.
|
||||
@return : count of most numerous element
|
||||
*/
|
||||
static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
@ -522,7 +518,6 @@ static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const BYTE* const end = ip + srcSize;
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned max=0;
|
||||
U32 s;
|
||||
|
||||
memset(count, 0, (maxSymbolValue+1)*sizeof(*count));
|
||||
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
||||
@ -532,7 +527,7 @@ static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
while (!count[maxSymbolValue]) maxSymbolValue--;
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
|
||||
for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s];
|
||||
{ U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; }
|
||||
|
||||
return (size_t)max;
|
||||
}
|
||||
@ -546,7 +541,6 @@ static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const BYTE* const iend = ip+sourceSize;
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned max=0;
|
||||
U32 s;
|
||||
|
||||
U32 Counting1[256] = { 0 };
|
||||
U32 Counting2[256] = { 0 };
|
||||
@ -561,8 +555,8 @@ static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
}
|
||||
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */
|
||||
|
||||
{ /* by stripes of 16 bytes */
|
||||
U32 cached = MEM_read32(ip); ip += 4;
|
||||
/* by stripes of 16 bytes */
|
||||
{ U32 cached = MEM_read32(ip); ip += 4;
|
||||
while (ip < iend-15) {
|
||||
U32 c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
@ -592,15 +586,15 @@ static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
while (ip<iend) Counting1[*ip++]++;
|
||||
|
||||
if (checkMax) { /* verify stats will fit into destination table */
|
||||
for (s=255; s>maxSymbolValue; s--) {
|
||||
U32 s; for (s=255; s>maxSymbolValue; s--) {
|
||||
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
|
||||
} }
|
||||
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
{ U32 s; for (s=0; s<=maxSymbolValue; s++) {
|
||||
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (count[s] > max) max = count[s];
|
||||
}
|
||||
}}
|
||||
|
||||
while (!count[maxSymbolValue]) maxSymbolValue--;
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
@ -634,7 +628,7 @@ size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
`U16 maxSymbolValue;`
|
||||
`U16 nextStateNumber[1 << tableLog];` // This size is variable
|
||||
`FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
|
||||
Allocation is manual, since C standard does not support variable-size structures.
|
||||
Allocation is manual (C standard does not support variable-size structures).
|
||||
*/
|
||||
|
||||
size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
|
||||
@ -730,7 +724,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
||||
/* all values are pretty poor;
|
||||
probably incompressible data (should have already been detected);
|
||||
find max, then give all remaining points to max */
|
||||
U32 maxV = 0, maxC =0;
|
||||
U32 maxV = 0, maxC = 0;
|
||||
for (s=0; s<=maxSymbolValue; s++)
|
||||
if (count[s] > maxC) maxV=s, maxC=count[s];
|
||||
norm[maxV] += (short)ToDistribute;
|
||||
@ -768,8 +762,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
|
||||
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
|
||||
|
||||
{
|
||||
U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
||||
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
||||
U64 const scale = 62 - tableLog;
|
||||
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
|
||||
U64 const vStep = 1ULL<<(scale-20);
|
||||
@ -845,13 +838,11 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
|
||||
tableU16[s] = (U16)(tableSize + s);
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{
|
||||
const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
|
||||
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
symbolTT[s].deltaNbBits = deltaNbBits;
|
||||
symbolTT[s].deltaFindState = s-1;
|
||||
}
|
||||
}
|
||||
} }
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -887,15 +878,13 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* ip=iend;
|
||||
|
||||
size_t errorCode;
|
||||
BIT_CStream_t bitC;
|
||||
FSE_CState_t CState1, CState2;
|
||||
|
||||
/* init */
|
||||
if (srcSize <= 2) return 0;
|
||||
errorCode = BIT_initCStream(&bitC, dst, dstSize);
|
||||
if (FSE_isError(errorCode)) return 0;
|
||||
{ size_t const errorCode = BIT_initCStream(&bitC, dst, dstSize);
|
||||
if (FSE_isError(errorCode)) return 0; }
|
||||
|
||||
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
|
||||
|
||||
@ -918,8 +907,7 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
|
||||
}
|
||||
|
||||
/* 2 or 4 encoding per loop */
|
||||
for ( ; ip>istart ; )
|
||||
{
|
||||
for ( ; ip>istart ; ) {
|
||||
FSE_encodeSymbol(&bitC, &CState2, *--ip);
|
||||
|
||||
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
|
||||
|
@ -267,7 +267,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
|
||||
BIT_flushBits(bitC);
|
||||
}
|
||||
|
||||
/* decompression */
|
||||
/*<===== Decompression =====>*/
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
@ -290,34 +290,39 @@ MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, con
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t FSE_getStateValue(FSE_DState_t* DStatePtr)
|
||||
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state;
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_peakSymbol(FSE_DState_t* DStatePtr)
|
||||
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
const U32 nbBits = DInfo.nbBits;
|
||||
BYTE symbol = DInfo.symbol;
|
||||
size_t lowBits = BIT_readBits(bitD, nbBits);
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
/*! FSE_decodeSymbolFast() :
|
||||
unsafe, only works if no symbol has a probability > 50% */
|
||||
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
const U32 nbBits = DInfo.nbBits;
|
||||
BYTE symbol = DInfo.symbol;
|
||||
size_t lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
|
141
lib/zbuff.c
141
lib/zbuff.c
@ -26,14 +26,9 @@
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not labelled stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
@ -124,21 +119,20 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params)
|
||||
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
size_t neededInBuffSize;
|
||||
|
||||
ZSTD_validateParams(¶ms);
|
||||
neededInBuffSize = (size_t)1 << params.windowLog;
|
||||
|
||||
/* allocate buffers */
|
||||
if (zbc->inBuffSize < neededInBuffSize) {
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
||||
if (zbc->inBuffSize < neededInBuffSize) {
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
|
||||
}
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, zbc->inBuffSize);
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
free(zbc->outBuff); /* should not be necessary */
|
||||
@ -146,26 +140,30 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dic
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
|
||||
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params);
|
||||
if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
|
||||
{ size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
zbc->inToCompress = 0;
|
||||
zbc->inBuffPos = 0;
|
||||
zbc->inBuffTarget = zbc->blockSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */
|
||||
zbc->stage = ZBUFFcs_load;
|
||||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
|
||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
|
||||
{
|
||||
return ZBUFF_compressInit_advanced(zbc, NULL, 0, ZSTD_getParams(compressionLevel, 0));
|
||||
}
|
||||
|
||||
|
||||
ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, ZSTD_getParams(compressionLevel, 0));
|
||||
return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
|
||||
}
|
||||
|
||||
|
||||
@ -185,11 +183,11 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
{
|
||||
U32 notDone = 1;
|
||||
const char* const istart = (const char*)src;
|
||||
const char* ip = istart;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
const char* ip = istart;
|
||||
char* const ostart = (char*)dst;
|
||||
char* op = ostart;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
char* op = ostart;
|
||||
|
||||
while (notDone) {
|
||||
switch(zbc->stage)
|
||||
@ -198,19 +196,17 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
|
||||
case ZBUFFcs_load:
|
||||
/* complete inBuffer */
|
||||
{
|
||||
size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
|
||||
{ size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
|
||||
zbc->inBuffPos += loaded;
|
||||
ip += loaded;
|
||||
if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
|
||||
notDone = 0; break; /* not enough input to get a full block : stop there, wait for more */
|
||||
} }
|
||||
/* compress current block (note : this stage cannot be stopped in the middle) */
|
||||
{
|
||||
void* cDst;
|
||||
{ void* cDst;
|
||||
size_t cSize;
|
||||
size_t iSize = zbc->inBuffPos - zbc->inToCompress;
|
||||
size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
|
||||
size_t oSize = oend-op;
|
||||
if (oSize >= ZSTD_compressBound(iSize))
|
||||
cDst = op; /* compress directly into output buffer (avoid flush stage) */
|
||||
@ -221,20 +217,18 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
/* prepare next block */
|
||||
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
|
||||
if (zbc->inBuffTarget > zbc->inBuffSize)
|
||||
{ zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; } /* note : inBuffSize >= blockSize */
|
||||
zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */
|
||||
zbc->inToCompress = zbc->inBuffPos;
|
||||
if (cDst == op) { op += cSize; break; } /* no need to flush */
|
||||
zbc->outBuffContentSize = cSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush;
|
||||
// break; /* flush stage follows */
|
||||
zbc->stage = ZBUFFcs_flush; /* continue to flush stage */
|
||||
}
|
||||
|
||||
case ZBUFFcs_flush:
|
||||
/* flush into dst */
|
||||
{
|
||||
size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
|
||||
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
|
||||
op += flushed;
|
||||
zbc->outBuffFlushedSize += flushed;
|
||||
if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
|
||||
@ -250,8 +244,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
|
||||
*srcSizePtr = ip - istart;
|
||||
*dstCapacityPtr = op - ostart;
|
||||
{
|
||||
size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
{ size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
if (hintInSize==0) hintInSize = zbc->blockSize;
|
||||
return hintInSize;
|
||||
}
|
||||
@ -271,7 +264,7 @@ size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
size_t srcSize = 0;
|
||||
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */
|
||||
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL */
|
||||
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
}
|
||||
|
||||
@ -279,11 +272,11 @@ size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + *dstCapacityPtr;
|
||||
BYTE* op = ostart;
|
||||
size_t outSize = *dstCapacityPtr;
|
||||
size_t epilogueSize, remaining;
|
||||
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
|
||||
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
|
||||
op += outSize;
|
||||
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
|
||||
zbc->outBuffContentSize += epilogueSize;
|
||||
@ -291,7 +284,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
zbc->stage = ZBUFFcs_flush;
|
||||
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
|
||||
op += outSize;
|
||||
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
|
||||
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
|
||||
*dstCapacityPtr = op-ostart; /* tells how many bytes were written */
|
||||
return remaining;
|
||||
}
|
||||
@ -328,10 +321,11 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
|
||||
struct ZBUFF_DCtx_s {
|
||||
ZSTD_DCtx* zc;
|
||||
ZSTD_frameParams fParams;
|
||||
char* inBuff;
|
||||
size_t blockSize;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
char* outBuff;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
@ -406,27 +400,26 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
return headerSize;
|
||||
} }
|
||||
|
||||
/* Frame header provides buffer sizes */
|
||||
{ size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */
|
||||
if (zbc->inBuffSize < neededInSize) {
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(1 << zbc->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
|
||||
zbc->blockSize = blockSize;
|
||||
if (zbc->inBuffSize < blockSize) {
|
||||
free(zbc->inBuff);
|
||||
zbc->inBuffSize = neededInSize;
|
||||
zbc->inBuff = (char*)malloc(neededInSize);
|
||||
zbc->inBuffSize = blockSize;
|
||||
zbc->inBuff = (char*)malloc(blockSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
{
|
||||
size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog;
|
||||
if (zbc->outBuffSize < neededOutSize) {
|
||||
free(zbc->outBuff);
|
||||
zbc->outBuffSize = neededOutSize;
|
||||
zbc->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
}
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbc->fParams.windowLog) + blockSize;
|
||||
if (zbc->outBuffSize < neededOutSize) {
|
||||
free(zbc->outBuff);
|
||||
zbc->outBuffSize = neededOutSize;
|
||||
zbc->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} } }
|
||||
zbc->stage = ZBUFFds_read;
|
||||
|
||||
case ZBUFFds_read:
|
||||
{
|
||||
size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
if (neededInSize==0) { /* end of frame */
|
||||
zbc->stage = ZBUFFds_init;
|
||||
notDone = 0;
|
||||
@ -449,8 +442,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
}
|
||||
|
||||
case ZBUFFds_load:
|
||||
{
|
||||
size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
@ -458,8 +450,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
ip += loadedSize;
|
||||
zbc->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||
{
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
|
||||
/* decode loaded input */
|
||||
{ size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
|
||||
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
|
||||
zbc->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
@ -469,15 +461,15 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
zbc->stage = ZBUFFds_flush;
|
||||
// break; /* ZBUFFds_flush follows */
|
||||
} }
|
||||
|
||||
case ZBUFFds_flush:
|
||||
{
|
||||
size_t const toFlushSize = zbc->outEnd - zbc->outStart;
|
||||
{ size_t const toFlushSize = zbc->outEnd - zbc->outStart;
|
||||
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zbc->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) {
|
||||
zbc->stage = ZBUFFds_read;
|
||||
if (zbc->outStart + ZSTD_BLOCKSIZE_MAX > zbc->outBuffSize)
|
||||
if (zbc->outStart + zbc->blockSize > zbc->outBuffSize)
|
||||
zbc->outStart = zbc->outEnd = 0;
|
||||
break;
|
||||
}
|
||||
@ -491,8 +483,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
/* result */
|
||||
*srcSizePtr = ip-istart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
{
|
||||
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||
nextSrcSizeHint -= zbc->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
|
13
lib/zbuff.h
13
lib/zbuff.h
@ -31,11 +31,6 @@
|
||||
#ifndef ZSTD_BUFFERED_H
|
||||
#define ZSTD_BUFFERED_H
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not considered stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
@ -89,12 +84,12 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
|
||||
* The nb of bytes written into @dst will be reported into *dstCapacityPtr.
|
||||
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
|
||||
* Note that the function cannot output more than *dstCapacityPtr,
|
||||
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
@ -139,13 +134,13 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
@ -51,7 +51,9 @@ extern "C" {
|
||||
/* *************************************
|
||||
* Advanced Streaming functions
|
||||
***************************************/
|
||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params);
|
||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
115
lib/zdict.c
115
lib/zdict.c
@ -284,8 +284,7 @@ static dictItem ZDICT_analyzePos(
|
||||
return solution;
|
||||
}
|
||||
|
||||
{
|
||||
int i;
|
||||
{ int i;
|
||||
U32 searchLength;
|
||||
U32 refinedStart = start;
|
||||
U32 refinedEnd = end;
|
||||
@ -575,7 +574,6 @@ static void ZDICT_fillNoise(void* buffer, size_t length)
|
||||
{
|
||||
unsigned acc = PRIME1;
|
||||
size_t p=0;;
|
||||
|
||||
for (p=0; p<length; p++) {
|
||||
acc *= PRIME2;
|
||||
((unsigned char*)buffer)[p] = (unsigned char)(acc >> 21);
|
||||
@ -595,29 +593,40 @@ static void ZDICT_countEStats(EStats_ress_t esr,
|
||||
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* bytePtr;
|
||||
const U32* u32Ptr;
|
||||
seqStore_t seqStore;
|
||||
const seqStore_t* seqStorePtr;
|
||||
|
||||
if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */
|
||||
ZSTD_copyCCtx(esr.zc, esr.ref);
|
||||
ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
|
||||
seqStore = ZSTD_copySeqStore(esr.zc);
|
||||
seqStorePtr = ZSTD_getSeqStore(esr.zc);
|
||||
|
||||
/* count stats */
|
||||
for(bytePtr = seqStore.litStart; bytePtr < seqStore.lit; bytePtr++)
|
||||
countLit[*bytePtr]++;
|
||||
for(u32Ptr = seqStore.offsetStart; u32Ptr < seqStore.offset; u32Ptr++) {
|
||||
BYTE offcode = (BYTE)ZSTD_highbit(*u32Ptr) + 1;
|
||||
if (*u32Ptr==0) offcode=0;
|
||||
offsetcodeCount[offcode]++;
|
||||
/* literals stats */
|
||||
{ const BYTE* bytePtr;
|
||||
for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
|
||||
countLit[*bytePtr]++;
|
||||
}
|
||||
for(bytePtr = seqStore.matchLengthStart; bytePtr < seqStore.matchLength; bytePtr++)
|
||||
matchlengthCount[*bytePtr]++;
|
||||
for(bytePtr = seqStore.litLengthStart; bytePtr < seqStore.litLength; bytePtr++)
|
||||
litlengthCount[*bytePtr]++;
|
||||
|
||||
/* seqStats */
|
||||
{ size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
|
||||
ZSTD_seqToCodes(seqStorePtr, nbSeq);
|
||||
|
||||
{ const BYTE* codePtr = seqStorePtr->offCodeStart;
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
|
||||
}
|
||||
|
||||
{ const BYTE* codePtr = seqStorePtr->mlCodeStart;
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
|
||||
}
|
||||
|
||||
{ const BYTE* codePtr = seqStorePtr->llCodeStart;
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
|
||||
} }
|
||||
}
|
||||
|
||||
/*
|
||||
static size_t ZDICT_maxSampleSize(const size_t* fileSizes, unsigned nbFiles)
|
||||
{
|
||||
unsigned u;
|
||||
@ -626,6 +635,15 @@ static size_t ZDICT_maxSampleSize(const size_t* fileSizes, unsigned nbFiles)
|
||||
if (max < fileSizes[u]) max = fileSizes[u];
|
||||
return max;
|
||||
}
|
||||
*/
|
||||
|
||||
static size_t ZDICT_totalSampleSize(const size_t* fileSizes, unsigned nbFiles)
|
||||
{
|
||||
size_t total;
|
||||
unsigned u;
|
||||
for (u=0, total=0; u<nbFiles; u++) total += fileSizes[u];
|
||||
return total;
|
||||
}
|
||||
|
||||
#define OFFCODE_MAX 18 /* only applicable to first block */
|
||||
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
@ -634,24 +652,26 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
U32 countLit[256];
|
||||
U32 offcodeCount[MaxOff+1];
|
||||
HUF_CREATE_STATIC_CTABLE(hufTable, 255);
|
||||
short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeCount[OFFCODE_MAX+1];
|
||||
short offcodeNCount[OFFCODE_MAX+1];
|
||||
U32 matchLengthCount[MaxML+1];
|
||||
short matchLengthNCount[MaxML+1];
|
||||
U32 litlengthCount[MaxLL+1];
|
||||
short litlengthNCount[MaxLL+1];
|
||||
U32 litLengthCount[MaxLL+1];
|
||||
short litLengthNCount[MaxLL+1];
|
||||
EStats_ress_t esr;
|
||||
ZSTD_parameters params;
|
||||
U32 u, huffLog = 12, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
||||
size_t pos = 0, errorCode;
|
||||
size_t eSize = 0;
|
||||
size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
|
||||
size_t const averageSampleSize = totalSrcSize / nbFiles;
|
||||
|
||||
/* init */
|
||||
for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */
|
||||
for (u=0; u<=OFFCODE_MAX; u++) offcodeCount[u]=1;
|
||||
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
|
||||
for (u=0; u<=MaxLL; u++) litlengthCount[u]=1;
|
||||
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
|
||||
esr.ref = ZSTD_createCCtx();
|
||||
esr.zc = ZSTD_createCCtx();
|
||||
esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
|
||||
@ -661,14 +681,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
goto _cleanup;
|
||||
}
|
||||
if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
|
||||
params = ZSTD_getParams(compressionLevel, MAX(dictBufferSize, ZDICT_maxSampleSize(fileSizes, nbFiles)));
|
||||
params.strategy = ZSTD_greedy;
|
||||
ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params);
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, averageSampleSize, dictBufferSize);
|
||||
params.cParams.strategy = ZSTD_greedy;
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
|
||||
|
||||
/* collect stats on all files */
|
||||
for (u=0; u<nbFiles; u++) {
|
||||
ZDICT_countEStats(esr,
|
||||
countLit, offcodeCount, matchLengthCount, litlengthCount,
|
||||
countLit, offcodeCount, matchLengthCount, litLengthCount,
|
||||
(const char*)srcBuffer + pos, fileSizes[u]);
|
||||
pos += fileSizes[u];
|
||||
}
|
||||
@ -700,11 +721,11 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
}
|
||||
mlLog = (U32)errorCode;
|
||||
|
||||
total=0; for (u=0; u<=MaxLL; u++) total+=litlengthCount[u];
|
||||
errorCode = FSE_normalizeCount(litlengthNCount, llLog, litlengthCount, total, MaxLL);
|
||||
total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
|
||||
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_normalizeCount error with litlengthCount");
|
||||
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
llLog = (U32)errorCode;
|
||||
@ -740,7 +761,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
maxDstSize -= errorCode;
|
||||
eSize += errorCode;
|
||||
|
||||
errorCode = FSE_writeNCount(dstBuffer, maxDstSize, litlengthNCount, MaxLL, llLog);
|
||||
errorCode = FSE_writeNCount(dstBuffer, maxDstSize, litLengthNCount, MaxLL, llLog);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount");
|
||||
@ -799,12 +820,17 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
|
||||
}
|
||||
|
||||
|
||||
#define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
|
||||
/*! ZDICT_trainFromBuffer_unsafe() :
|
||||
* `samplesBuffer` must be followed by noisy guard band.
|
||||
* @return : size of dictionary.
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_unsafe(
|
||||
void* dictBuffer, size_t maxDictSize,
|
||||
const void* samplesBuffer, const size_t* sampleSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
const U32 dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
|
||||
U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
|
||||
dictItem* dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
|
||||
unsigned selectivity = params.selectivityLevel;
|
||||
unsigned compressionLevel = params.compressionLevel;
|
||||
@ -814,10 +840,11 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
|
||||
/* checks */
|
||||
if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) return ERROR(dstSize_tooSmall);
|
||||
if (!dictList) return ERROR(memory_allocation);
|
||||
|
||||
/* init */
|
||||
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
|
||||
if (!dictList) return ERROR(memory_allocation);
|
||||
if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */
|
||||
ZDICT_initDictItem(dictList);
|
||||
g_displayLevel = params.notificationLevel;
|
||||
if (selectivity==0) selectivity = g_selectivity_default;
|
||||
@ -832,9 +859,9 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
|
||||
/* display best matches */
|
||||
if (g_displayLevel>= 3) {
|
||||
const U32 nb = 25;
|
||||
U32 const nb = 25;
|
||||
U32 const dictContentSize = ZDICT_dictSize(dictList);
|
||||
U32 u;
|
||||
U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
|
||||
DISPLAYLEVEL(3, "list %u best segments \n", nb);
|
||||
for (u=1; u<=nb; u++) {
|
||||
@ -848,8 +875,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
} } }
|
||||
|
||||
/* create dictionary */
|
||||
{
|
||||
U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||
{ U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||
size_t hSize;
|
||||
BYTE* ptr;
|
||||
U32 u;
|
||||
@ -894,31 +920,32 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
}
|
||||
|
||||
|
||||
/* issue : samplesBuffer need to be followed by a noisy guard band.
|
||||
* work around : duplicate the buffer, and add the noise */
|
||||
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
size_t sBuffSize;
|
||||
void* newBuff;
|
||||
size_t result;
|
||||
size_t sBuffSize;
|
||||
|
||||
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
|
||||
if (sBuffSize==0) return 0; /* empty content => no dictionary */
|
||||
newBuff = malloc(sBuffSize + NOISELENGTH);
|
||||
if (!newBuff) return ERROR(memory_allocation);
|
||||
|
||||
memcpy(newBuff, samplesBuffer, sBuffSize);
|
||||
ZDICT_fillNoise((char*)newBuff + sBuffSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
||||
|
||||
result = ZDICT_trainFromBuffer_unsafe(dictBuffer, dictBufferCapacity,
|
||||
{ size_t const result = ZDICT_trainFromBuffer_unsafe(
|
||||
dictBuffer, dictBufferCapacity,
|
||||
newBuff, samplesSizes, nbSamples,
|
||||
params);
|
||||
free(newBuff);
|
||||
return result;
|
||||
free(newBuff);
|
||||
return result; }
|
||||
}
|
||||
|
||||
|
||||
/* issue : samplesBuffer need to be followed by a noisy guard band.
|
||||
* work around : duplicate the buffer, and add the noise ? */
|
||||
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
|
||||
{
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -325,7 +325,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
|
||||
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
|
||||
size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
|
||||
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
return result;
|
||||
}
|
||||
@ -503,7 +503,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
@return : nb bytes read from src,
|
||||
or an error code if it fails, testable with ZSTD_isError()
|
||||
*/
|
||||
FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog,
|
||||
FORCE_INLINE size_t ZSTD_buildSeqTableOff(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 max = (1<<rawBits)-1;
|
||||
@ -531,63 +531,80 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
||||
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
||||
const void* src, size_t srcSize)
|
||||
FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
||||
const void* src, size_t srcSize,
|
||||
const S16* defaultNorm, U32 defaultLog)
|
||||
{
|
||||
switch(type)
|
||||
{
|
||||
case FSE_ENCODING_RLE :
|
||||
if (!srcSize) return ERROR(srcSize_wrong);
|
||||
if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected);
|
||||
FSE_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */
|
||||
return 1;
|
||||
case FSE_ENCODING_RAW :
|
||||
FSE_buildDTable(DTable, defaultNorm, max, defaultLog);
|
||||
return 0;
|
||||
case FSE_ENCODING_STATIC:
|
||||
return 0;
|
||||
default : /* impossible */
|
||||
case FSE_ENCODING_DYNAMIC :
|
||||
{ U32 tableLog;
|
||||
S16 norm[MaxSeq+1];
|
||||
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
|
||||
if (FSE_isError(headerSize)) return ERROR(corruption_detected);
|
||||
if (tableLog > maxLog) return ERROR(corruption_detected);
|
||||
FSE_buildDTable(DTable, norm, max, tableLog);
|
||||
return headerSize;
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
|
||||
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE* const)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
U32 LLtype, Offtype, MLtype;
|
||||
const BYTE* ip = istart;
|
||||
|
||||
/* check */
|
||||
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
|
||||
|
||||
/* SeqHead */
|
||||
*nbSeq = *ip++;
|
||||
if (*nbSeq==0) return 1;
|
||||
if (*nbSeq >= 0x7F) {
|
||||
if (*nbSeq == 0xFF)
|
||||
*nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
|
||||
else
|
||||
*nbSeq = ((nbSeq[0]-0x80)<<8) + *ip++;
|
||||
{ int nbSeq = *ip++;
|
||||
if (!nbSeq) { *nbSeqPtr=0; return 1; }
|
||||
if (nbSeq > 0x7F) {
|
||||
if (nbSeq == 0xFF)
|
||||
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
|
||||
else
|
||||
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
|
||||
}
|
||||
*nbSeqPtr = nbSeq;
|
||||
}
|
||||
|
||||
/* FSE table descriptors */
|
||||
LLtype = *ip >> 6;
|
||||
Offtype = (*ip >> 4) & 3;
|
||||
MLtype = (*ip >> 2) & 3;
|
||||
{ size_t dumpsLength;
|
||||
if (*ip & 2) {
|
||||
dumpsLength = ip[2];
|
||||
dumpsLength += ip[1] << 8;
|
||||
ip += 3;
|
||||
} else {
|
||||
dumpsLength = ip[1];
|
||||
dumpsLength += (ip[0] & 1) << 8;
|
||||
ip += 2;
|
||||
{ U32 const LLtype = *ip >> 6;
|
||||
U32 const Offtype = (*ip >> 4) & 3;
|
||||
U32 const MLtype = (*ip >> 2) & 3;
|
||||
ip++;
|
||||
|
||||
/* check */
|
||||
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
|
||||
/* Build DTables */
|
||||
{ size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog);
|
||||
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
}
|
||||
*dumpsPtr = ip;
|
||||
ip += dumpsLength;
|
||||
*dumpsLengthPtr = dumpsLength;
|
||||
}
|
||||
|
||||
/* check */
|
||||
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
|
||||
/* Build DTables */
|
||||
{ size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, LLbits, LLFSELog, ip, iend-ip);
|
||||
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
}
|
||||
{ size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, Offbits, OffFSELog, ip, iend-ip);
|
||||
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
}
|
||||
{ size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MLbits, MLFSELog, ip, iend-ip);
|
||||
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
}
|
||||
{ size_t const bhSize = ZSTD_buildSeqTableOff(DTableOffb, Offtype, Offbits, OffFSELog, ip, iend-ip);
|
||||
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
}
|
||||
{ size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog);
|
||||
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
} }
|
||||
|
||||
return ip-istart;
|
||||
}
|
||||
@ -605,78 +622,58 @@ typedef struct {
|
||||
FSE_DState_t stateOffb;
|
||||
FSE_DState_t stateML;
|
||||
size_t prevOffset;
|
||||
const BYTE* dumps;
|
||||
const BYTE* dumpsEnd;
|
||||
} seqState_t;
|
||||
|
||||
|
||||
static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
|
||||
{
|
||||
const BYTE* dumps = seqState->dumps;
|
||||
const BYTE* const de = seqState->dumpsEnd;
|
||||
size_t litLength, offset;
|
||||
|
||||
/* Literal length */
|
||||
litLength = FSE_peakSymbol(&(seqState->stateLL));
|
||||
if (litLength == MaxLL) {
|
||||
const U32 add = *dumps++;
|
||||
if (add < 255) litLength += add;
|
||||
else {
|
||||
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no risk : dumps is always followed by seq tables > 1 byte */
|
||||
if (litLength&1) litLength>>=1, dumps += 3;
|
||||
else litLength = (U16)(litLength)>>1, dumps += 2;
|
||||
}
|
||||
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
||||
}
|
||||
U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
|
||||
U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
|
||||
U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||
|
||||
/* Offset */
|
||||
{ static const U32 offsetPrefix[MaxOff+1] = {
|
||||
1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40,
|
||||
0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000,
|
||||
0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000,
|
||||
0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 };
|
||||
U32 const offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||
U32 const nbBits = offsetCode ? offsetCode-1 : 0;
|
||||
offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits);
|
||||
U32 const llBits = LL_bits[llCode];
|
||||
U32 const mlBits = ML_bits[mlCode];
|
||||
U32 const ofBits = ofCode;
|
||||
U32 const totalBits = llBits+mlBits+ofBits;
|
||||
|
||||
static const U32 LL_base[MaxLL+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
|
||||
0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
|
||||
static const U32 ML_base[MaxML+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800,
|
||||
0x1000, 0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
|
||||
static const U32 OF_base[MaxOff+1] = {
|
||||
0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
|
||||
0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF,
|
||||
0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
|
||||
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 };
|
||||
|
||||
/* sequence */
|
||||
{ size_t const offset = ofCode ? OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits) : /* <= 26 bits */
|
||||
llCode ? seq->offset : seqState->prevOffset;
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset;
|
||||
if (offsetCode | !litLength) seqState->prevOffset = seq->offset; /* cmove */
|
||||
FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* update */
|
||||
if (ofCode | !llCode) seqState->prevOffset = seq->offset; /* cmove */
|
||||
seq->offset = offset;
|
||||
}
|
||||
|
||||
/* Literal length update */
|
||||
FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream)); /* update */
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
seq->matchLength = ML_base[mlCode] + mls + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
|
||||
|
||||
/* MatchLength */
|
||||
{ size_t matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
||||
if (matchLength == MaxML) {
|
||||
const U32 add = *dumps++;
|
||||
if (add < 255) matchLength += add;
|
||||
else {
|
||||
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
if (matchLength&1) matchLength>>=1, dumps += 3;
|
||||
else matchLength = (U16)(matchLength)>>1, dumps += 2;
|
||||
}
|
||||
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
||||
}
|
||||
matchLength += mls;
|
||||
seq->matchLength = matchLength;
|
||||
}
|
||||
seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() ||
|
||||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
|
||||
|
||||
/* save result */
|
||||
seq->litLength = litLength;
|
||||
seq->offset = offset;
|
||||
seqState->dumps = dumps;
|
||||
|
||||
#if 0 /* debug */
|
||||
{
|
||||
static U64 totalDecoded = 0;
|
||||
printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
|
||||
(U32)(totalDecoded), (U32)litLength, (U32)matchLength, (U32)offset);
|
||||
totalDecoded += litLength + matchLength;
|
||||
}
|
||||
#endif
|
||||
/* ANS state update */
|
||||
FSE_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */
|
||||
FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
||||
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
||||
}
|
||||
|
||||
|
||||
@ -685,30 +682,27 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
|
||||
const BYTE** litPtr, const BYTE* const litLimit_8,
|
||||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
{
|
||||
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
const size_t sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
BYTE* const oend_8 = oend-8;
|
||||
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* match = oLitEnd - sequence.offset;
|
||||
|
||||
/* check */
|
||||
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
||||
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
|
||||
if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
*litPtr = iLitEnd; /* update for next sequence */
|
||||
|
||||
/* copy Match */
|
||||
if (sequence.offset > (size_t)(oLitEnd - base)) {
|
||||
/* offset beyond prefix */
|
||||
if (sequence.offset > (size_t)(oLitEnd - vBase))
|
||||
return ERROR(corruption_detected);
|
||||
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
|
||||
match = dictEnd - (base-match);
|
||||
if (match + sequence.matchLength <= dictEnd) {
|
||||
memmove(oLitEnd, match, sequence.matchLength);
|
||||
@ -725,7 +719,9 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
|
||||
/* match within prefix */
|
||||
if (sequence.offset < 8) {
|
||||
/* close range match, overlap */
|
||||
const int sub2 = dec64table[sequence.offset];
|
||||
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
int const sub2 = dec64table[sequence.offset];
|
||||
op[0] = match[0];
|
||||
op[1] = match[1];
|
||||
op[2] = match[2];
|
||||
@ -762,12 +758,9 @@ static size_t ZSTD_decompressSequences(
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
size_t dumpsLength;
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
int nbSeq;
|
||||
const BYTE* dumps;
|
||||
U32* DTableLL = dctx->LLTable;
|
||||
U32* DTableML = dctx->MLTable;
|
||||
U32* DTableOffb = dctx->OffTable;
|
||||
@ -775,13 +768,14 @@ static size_t ZSTD_decompressSequences(
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
const U32 mls = dctx->fParams.mml;
|
||||
int nbSeq;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
{ size_t const errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
|
||||
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq,
|
||||
DTableLL, DTableML, DTableOffb,
|
||||
ip, seqSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
ip += errorCode;
|
||||
if (ZSTD_isError(seqHSize)) return seqHSize;
|
||||
ip += seqHSize;
|
||||
}
|
||||
|
||||
/* Regen sequences */
|
||||
@ -791,8 +785,6 @@ static size_t ZSTD_decompressSequences(
|
||||
|
||||
memset(&sequence, 0, sizeof(sequence));
|
||||
sequence.offset = REPCODE_STARTVALUE;
|
||||
seqState.dumps = dumps;
|
||||
seqState.dumpsEnd = dumps + dumpsLength;
|
||||
seqState.prevOffset = REPCODE_STARTVALUE;
|
||||
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
|
||||
@ -804,6 +796,13 @@ static size_t ZSTD_decompressSequences(
|
||||
size_t oneSeqSize;
|
||||
nbSeq--;
|
||||
ZSTD_decodeSequence(&sequence, &seqState, mls);
|
||||
#if 0 /* for debug */
|
||||
{ U32 pos = (U32)(op-base);
|
||||
if ((pos > 200802300) && (pos < 200802400))
|
||||
printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
|
||||
}
|
||||
#endif
|
||||
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
@ -841,17 +840,16 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
||||
const void* src, size_t srcSize)
|
||||
{ /* blockType == blockCompressed */
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t litCSize;
|
||||
|
||||
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
|
||||
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->fParams.mml);
|
||||
|
||||
/* Decode literals sub-block */
|
||||
litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
if (ZSTD_isError(litCSize)) return litCSize;
|
||||
ip += litCSize;
|
||||
srcSize -= litCSize;
|
||||
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
if (ZSTD_isError(litCSize)) return litCSize;
|
||||
ip += litCSize;
|
||||
srcSize -= litCSize; }
|
||||
|
||||
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
|
||||
}
|
||||
@ -900,7 +898,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
size_t decodedSize=0;
|
||||
size_t cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
@ -1000,7 +998,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
{
|
||||
/* get frame header size */
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
@ -1014,7 +1011,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
}
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
{
|
||||
/* get frame header */
|
||||
size_t result;
|
||||
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
|
||||
result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
@ -1025,16 +1021,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
}
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
{
|
||||
/* Decode block header */
|
||||
blockProperties_t bp;
|
||||
size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(blockSize)) return blockSize;
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
if (bp.blockType == bt_end) {
|
||||
dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
}
|
||||
else {
|
||||
dctx->expected = blockSize;
|
||||
} else {
|
||||
dctx->expected = cBlockSize;
|
||||
dctx->bType = bp.blockType;
|
||||
dctx->stage = ZSTDds_decompressBlock;
|
||||
}
|
||||
@ -1121,7 +1115,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
|
||||
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
size_t eSize;
|
||||
U32 magic = MEM_readLE32(dict);
|
||||
U32 const magic = MEM_readLE32(dict);
|
||||
if (magic != ZSTD_DICT_MAGIC) {
|
||||
/* pure content mode */
|
||||
ZSTD_refDictContent(dctx, dict, dictSize);
|
||||
@ -1144,12 +1138,11 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
|
||||
|
||||
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
size_t errorCode;
|
||||
errorCode = ZSTD_decompressBegin(dctx);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
{ size_t const errorCode = ZSTD_decompressBegin(dctx);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
if (dict && dictSize) {
|
||||
errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
|
||||
size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
|
||||
if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
}
|
||||
|
||||
|
@ -99,23 +99,39 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
#define REPCODE_STARTVALUE 1
|
||||
|
||||
#define Litbits 8
|
||||
#define MLbits 7
|
||||
#define LLbits 6
|
||||
#define Offbits 5
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
#define MaxML ((1<<MLbits) - 1)
|
||||
#define MaxLL ((1<<LLbits) - 1)
|
||||
#define MaxML 52
|
||||
#define MaxLL 35
|
||||
#define MaxOff ((1<<Offbits)- 1)
|
||||
#define MLFSELog 10
|
||||
#define LLFSELog 10
|
||||
#define OffFSELog 9
|
||||
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
|
||||
#define MLFSELog 9
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
|
||||
#define FSE_ENCODING_RAW 0
|
||||
#define FSE_ENCODING_RLE 1
|
||||
#define FSE_ENCODING_STATIC 2
|
||||
#define FSE_ENCODING_DYNAMIC 3
|
||||
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1,
|
||||
1, 1, 1, 1 };
|
||||
static const U32 LL_defaultNormLog = 6;
|
||||
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, };
|
||||
static const U32 ML_defaultNormLog = 6;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Shared functions to include for inlining
|
||||
@ -191,15 +207,15 @@ typedef struct {
|
||||
U32* offsetStart;
|
||||
U32* offset;
|
||||
BYTE* offCodeStart;
|
||||
BYTE* offCode;
|
||||
BYTE* litStart;
|
||||
BYTE* lit;
|
||||
BYTE* litLengthStart;
|
||||
BYTE* litLength;
|
||||
BYTE* matchLengthStart;
|
||||
BYTE* matchLength;
|
||||
BYTE* dumpsStart;
|
||||
BYTE* dumps;
|
||||
U16* litLengthStart;
|
||||
U16* litLength;
|
||||
BYTE* llCodeStart;
|
||||
U16* matchLengthStart;
|
||||
U16* matchLength;
|
||||
BYTE* mlCodeStart;
|
||||
U32 longLength;
|
||||
/* opt */
|
||||
ZSTD_optimal_t* priceTable;
|
||||
ZSTD_match_t* matchTable;
|
||||
@ -221,7 +237,9 @@ typedef struct {
|
||||
ZSTD_stats_t stats;
|
||||
} seqStore_t;
|
||||
|
||||
seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx);
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel);
|
||||
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
@ -31,7 +31,7 @@
|
||||
- Zstd source repository : https://www.zstd.net
|
||||
*/
|
||||
|
||||
/* Note : this file is intended to be included within zstd_compress.c */
|
||||
/* Note : this file is intended to be included within zstd_compress.c */
|
||||
|
||||
|
||||
#define ZSTD_FREQ_DIV 5
|
||||
@ -55,8 +55,8 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
|
||||
|
||||
if (ssPtr->litLengthSum == 0) {
|
||||
ssPtr->litSum = (2<<Litbits);
|
||||
ssPtr->litLengthSum = (1<<LLbits);
|
||||
ssPtr->matchLengthSum = (1<<MLbits);
|
||||
ssPtr->litLengthSum = MaxLL+1;
|
||||
ssPtr->matchLengthSum = MaxML+1;
|
||||
ssPtr->offCodeSum = (1<<Offbits);
|
||||
ssPtr->matchSum = (2<<Litbits);
|
||||
|
||||
@ -93,7 +93,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
|
||||
ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ZSTD_setLog2Prices(ssPtr);
|
||||
}
|
||||
|
||||
@ -219,12 +219,12 @@ static U32 ZSTD_insertBtAndGetAllMatches (
|
||||
{
|
||||
const BYTE* const base = zc->base;
|
||||
const U32 current = (U32)(ip-base);
|
||||
const U32 hashLog = zc->params.hashLog;
|
||||
const U32 hashLog = zc->params.cParams.hashLog;
|
||||
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
|
||||
U32* const hashTable = zc->hashTable;
|
||||
U32 matchIndex = hashTable[h];
|
||||
U32* const bt = zc->contentTable;
|
||||
const U32 btLog = zc->params.contentLog - 1;
|
||||
const U32 btLog = zc->params.cParams.contentLog - 1;
|
||||
const U32 btMask= (1U << btLog) - 1;
|
||||
size_t commonLengthSmaller=0, commonLengthLarger=0;
|
||||
const BYTE* const dictBase = zc->dictBase;
|
||||
@ -244,7 +244,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
|
||||
|
||||
if (minMatch == 3) { /* HC3 match finder */
|
||||
U32 matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip);
|
||||
|
||||
|
||||
if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) {
|
||||
const BYTE* match;
|
||||
size_t currentMl=0;
|
||||
@ -409,12 +409,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ctx->base;
|
||||
const BYTE* const prefixStart = base + ctx->dictLimit;
|
||||
|
||||
|
||||
U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE;
|
||||
const U32 maxSearches = 1U << ctx->params.searchLog;
|
||||
const U32 sufficient_len = ctx->params.targetLength;
|
||||
const U32 mls = ctx->params.searchLength;
|
||||
const U32 minMatch = (ctx->params.searchLength == 3) ? 3 : 4;
|
||||
const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
|
||||
const U32 sufficient_len = ctx->params.cParams.targetLength;
|
||||
const U32 mls = ctx->params.cParams.searchLength;
|
||||
const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
|
||||
|
||||
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
|
||||
ZSTD_match_t* matches = seqStorePtr->matchTable;
|
||||
@ -734,12 +734,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const dictBase = ctx->dictBase;
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
const U32 lowLimit = ctx->lowLimit;
|
||||
|
||||
|
||||
U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE;
|
||||
const U32 maxSearches = 1U << ctx->params.searchLog;
|
||||
const U32 sufficient_len = ctx->params.targetLength;
|
||||
const U32 mls = ctx->params.searchLength;
|
||||
const U32 minMatch = (ctx->params.searchLength == 3) ? 3 : 4;
|
||||
const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
|
||||
const U32 sufficient_len = ctx->params.cParams.targetLength;
|
||||
const U32 mls = ctx->params.cParams.searchLength;
|
||||
const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
|
||||
|
||||
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
|
||||
ZSTD_match_t* matches = seqStorePtr->matchTable;
|
||||
@ -1045,12 +1045,12 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
|
||||
break;
|
||||
} else {
|
||||
const BYTE* repMatch = dictBase + ((anchor-base) - rep_2);
|
||||
if ((repMatch + minMatch <= dictEnd) && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch)))
|
||||
if ((repMatch + minMatch <= dictEnd) && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch)))
|
||||
mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, dictEnd, prefixStart) + minMatch;
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */
|
||||
ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2);
|
||||
ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch);
|
||||
|
@ -57,7 +57,7 @@ extern "C" {
|
||||
/*-*************************************
|
||||
* Types
|
||||
***************************************/
|
||||
#define ZSTD_WINDOWLOG_MAX 27
|
||||
#define ZSTD_WINDOWLOG_MAX (MEM_32bits() ? 25 : 27)
|
||||
#define ZSTD_WINDOWLOG_MIN 18
|
||||
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_CONTENTLOG_MIN 4
|
||||
@ -76,7 +76,6 @@ extern "C" {
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;
|
||||
|
||||
typedef struct {
|
||||
U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */
|
||||
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
|
||||
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /* dispatch table : larger == faster, more memory */
|
||||
@ -84,6 +83,15 @@ typedef struct {
|
||||
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
|
||||
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
|
||||
ZSTD_strategy strategy;
|
||||
} ZSTD_compressionParameters;
|
||||
|
||||
typedef struct {
|
||||
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
|
||||
} ZSTD_frameParameters;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
} ZSTD_parameters;
|
||||
|
||||
|
||||
@ -92,14 +100,19 @@ typedef struct {
|
||||
***************************************/
|
||||
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
||||
|
||||
/*! ZSTD_getParams() :
|
||||
* @return ZSTD_parameters structure for a selected compression level and srcSize.
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize);
|
||||
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_validateParams() :
|
||||
* correct params value to remain within authorized range */
|
||||
ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params);
|
||||
/*! ZSTD_checkParams() :
|
||||
* Ensure param values remain within authorized range */
|
||||
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||
|
||||
/*! ZSTD_adjustParams() :
|
||||
* optimize params for a given `srcSize` and `dictSize`.
|
||||
* both values are optional, select `0` if unknown. */
|
||||
ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_compress_advanced() :
|
||||
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
||||
@ -137,7 +150,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
|
||||
****************************************/
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
148
programs/bench.c
148
programs/bench.c
@ -58,16 +58,16 @@
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# define BMK_sleep(s) sleep(s)
|
||||
# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000ULL; nanosleep(&t, NULL); }
|
||||
# define setHighPriority() setpriority(PRIO_PROCESS, 0, -20)
|
||||
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
#elif defined(_WIN32)
|
||||
# include <windows.h>
|
||||
# define BMK_sleep(s) Sleep(1000*s)
|
||||
# define mili_sleep(mili) Sleep(mili)
|
||||
# define setHighPriority() SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
#else
|
||||
# define BMK_sleep(s) /* disabled */
|
||||
# define mili_sleep(mili) /* disabled */
|
||||
# define setHighPriority() /* disabled */
|
||||
# define SET_HIGH_PRIORITY /* disabled */
|
||||
#endif
|
||||
|
||||
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
@ -97,6 +97,7 @@
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Compiler specifics
|
||||
***************************************/
|
||||
@ -238,18 +239,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, U32 nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize, benchResult_t *result)
|
||||
{
|
||||
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
|
||||
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
ZSTD_CCtx* refCtx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
U32 nbBlocks;
|
||||
BMK_time_t ticksPerSecond;
|
||||
|
||||
@ -287,20 +286,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
|
||||
|
||||
/* Bench */
|
||||
{ size_t cSize = 0;
|
||||
double fastestC = 100000000., fastestD = 100000000.;
|
||||
double ratio = 0.;
|
||||
{ double fastestC = 100000000., fastestD = 100000000.;
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
U64 crcCheck = 0;
|
||||
BMK_time_t coolTime;
|
||||
U32 testNb;
|
||||
size_t cSize = 0;
|
||||
double ratio = 0.;
|
||||
|
||||
BMK_getTime(coolTime);
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
|
||||
int nbLoops;
|
||||
BMK_time_t clockStart, clockEnd;
|
||||
U64 clockSpan;
|
||||
U64 const clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10;
|
||||
U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10;
|
||||
|
||||
/* overheat protection */
|
||||
if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) {
|
||||
@ -319,19 +317,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
|
||||
BMK_getTime(clockStart);
|
||||
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
|
||||
// ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
} }
|
||||
{ U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
|
||||
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
|
||||
|
||||
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
@ -339,6 +340,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC );
|
||||
|
||||
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
#if 1
|
||||
/* Decompression */
|
||||
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
|
||||
@ -349,53 +351,55 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
|
||||
BMK_getTime(clockStart);
|
||||
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
|
||||
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
if (ZSTD_isError(regenSize)) {
|
||||
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s",
|
||||
blockNb, ZSTD_getErrorName(regenSize));
|
||||
goto _findError;
|
||||
}
|
||||
blockTable[blockNb].resSize = regenSize;
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
|
||||
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
if (ZSTD_isError(regenSize)) {
|
||||
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
|
||||
blockNb, ZSTD_getErrorName(regenSize));
|
||||
clockLoop = 0; /* force immediate test end */
|
||||
break;
|
||||
}
|
||||
blockTable[blockNb].resSize = regenSize;
|
||||
} }
|
||||
{ U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
|
||||
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
|
||||
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC,
|
||||
(double)srcSize / fastestD );
|
||||
|
||||
/* CRC Checking */
|
||||
_findError:
|
||||
crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
|
||||
U32 segNb, bNb, pos;
|
||||
size_t bacc = 0;
|
||||
printf("Decoding error at pos %u ", (U32)u);
|
||||
for (segNb = 0; segNb < nbBlocks; segNb++) {
|
||||
if (bacc + blockTable[segNb].srcSize > u) break;
|
||||
bacc += blockTable[segNb].srcSize;
|
||||
{ crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
|
||||
U32 segNb, bNb, pos;
|
||||
size_t bacc = 0;
|
||||
DISPLAY("Decoding error at pos %u ", (U32)u);
|
||||
for (segNb = 0; segNb < nbBlocks; segNb++) {
|
||||
if (bacc + blockTable[segNb].srcSize > u) break;
|
||||
bacc += blockTable[segNb].srcSize;
|
||||
}
|
||||
pos = (U32)(u - bacc);
|
||||
bNb = pos / (128 KB);
|
||||
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
|
||||
break;
|
||||
}
|
||||
pos = (U32)(u - bacc);
|
||||
bNb = pos / (128 KB);
|
||||
printf("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
|
||||
break;
|
||||
}
|
||||
if (u==srcSize-1) { /* should never happen */
|
||||
printf("no difference detected\n");
|
||||
} }
|
||||
break;
|
||||
} /* if (crcOrig!=crcCheck) */
|
||||
if (u==srcSize-1) { /* should never happen */
|
||||
DISPLAY("no difference detected\n");
|
||||
} }
|
||||
break;
|
||||
} } /* CRC Checking */
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
|
||||
|
||||
@ -445,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
benchResult_t result, total;
|
||||
int l;
|
||||
|
||||
setHighPriority();
|
||||
SET_HIGH_PRIORITY;
|
||||
|
||||
const char* pch = strrchr(displayName, '\\'); /* Windows */
|
||||
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
|
||||
@ -494,23 +498,25 @@ static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
return total;
|
||||
}
|
||||
|
||||
/*! BMK_loadFiles() :
|
||||
Loads `buffer` with content of files listed within `fileNamesTable`.
|
||||
At most, fills `buffer` entirely */
|
||||
static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned const nbFiles)
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0;
|
||||
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
size_t readSize;
|
||||
U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
|
||||
FILE* f = fopen(fileNamesTable[n], "rb");
|
||||
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
|
||||
readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize;
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize; }
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
|
@ -333,28 +333,28 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
{
|
||||
FILE* srcFile = ress.srcFile;
|
||||
FILE* dstFile = ress.dstFile;
|
||||
U64 filesize = 0;
|
||||
U64 readsize = 0;
|
||||
U64 compressedfilesize = 0;
|
||||
size_t dictSize = ress.dictBufferSize;
|
||||
size_t sizeCheck, errorCode;
|
||||
size_t sizeCheck;
|
||||
ZSTD_parameters params;
|
||||
U64 const fileSize = FIO_getFileSize(srcFileName);
|
||||
|
||||
/* init */
|
||||
filesize = MAX(FIO_getFileSize(srcFileName),dictSize);
|
||||
params = ZSTD_getParams(cLevel, filesize);
|
||||
params.srcSize = filesize;
|
||||
if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog;
|
||||
errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params);
|
||||
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
|
||||
params.cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
|
||||
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
||||
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
|
||||
|
||||
/* Main compression loop */
|
||||
filesize = 0;
|
||||
readsize = 0;
|
||||
while (1) {
|
||||
/* Fill input Buffer */
|
||||
size_t inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
|
||||
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
|
||||
if (inSize==0) break;
|
||||
filesize += inSize;
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
|
||||
readsize += inSize;
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20));
|
||||
|
||||
{ /* Compress using buffered streaming */
|
||||
size_t usedInSize = inSize;
|
||||
@ -371,13 +371,12 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
|
||||
compressedfilesize += cSize;
|
||||
}
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
|
||||
}
|
||||
|
||||
/* End of Frame */
|
||||
{
|
||||
size_t cSize = ress.dstBufferSize;
|
||||
size_t result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
|
||||
{ size_t cSize = ress.dstBufferSize;
|
||||
size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
|
||||
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
|
||||
|
||||
sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
|
||||
@ -388,7 +387,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
/* Status */
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
|
||||
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
|
||||
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (double)compressedfilesize/readsize*100);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -118,7 +118,7 @@ static clock_t BMK_clockSpan( clock_t clockStart )
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
const size_t step = 64 MB;
|
||||
size_t const step = 64 MB;
|
||||
void* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
@ -182,15 +182,13 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
|
||||
}
|
||||
|
||||
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
|
||||
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
|
||||
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
|
||||
const BYTE* dumps;
|
||||
size_t length;
|
||||
int nbSeq;
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
|
||||
return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
|
||||
}
|
||||
|
||||
|
||||
|
@ -488,12 +488,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
|
||||
if (cSize > 3) {
|
||||
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
||||
const size_t tooSmallSize = cSize - missing;
|
||||
static const U32 endMark = 0x4DC2B1A9;
|
||||
const U32 endMark = 0x4DC2B1A9;
|
||||
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
|
||||
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
|
||||
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
|
||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
|
||||
}
|
||||
|
||||
/* frame header decompression test */
|
||||
|
@ -126,7 +126,7 @@ static U32 g_rand = 1;
|
||||
static U32 g_singleRun = 0;
|
||||
static U32 g_target = 0;
|
||||
static U32 g_noSeed = 0;
|
||||
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, 0, ZSTD_greedy };
|
||||
static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
@ -251,7 +251,7 @@ typedef struct
|
||||
static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx,
|
||||
const ZSTD_parameters params)
|
||||
const ZSTD_compressionParameters cParams)
|
||||
{
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
|
||||
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
|
||||
@ -259,13 +259,14 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
U32 Wlog = params.windowLog;
|
||||
U32 Clog = params.contentLog;
|
||||
U32 Hlog = params.hashLog;
|
||||
U32 Slog = params.searchLog;
|
||||
U32 Slength = params.searchLength;
|
||||
U32 Tlength = params.targetLength;
|
||||
ZSTD_strategy strat = params.strategy;
|
||||
ZSTD_parameters params;
|
||||
U32 Wlog = cParams.windowLog;
|
||||
U32 Clog = cParams.contentLog;
|
||||
U32 Hlog = cParams.hashLog;
|
||||
U32 Slog = cParams.searchLog;
|
||||
U32 Slength = cParams.searchLength;
|
||||
U32 Tlength = cParams.targetLength;
|
||||
ZSTD_strategy strat = cParams.strategy;
|
||||
char name[30] = { 0 };
|
||||
U64 crcOrig;
|
||||
|
||||
@ -315,6 +316,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const int startTime =BMK_GetMilliStart();
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
params.cParams = cParams;
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
int nbLoops;
|
||||
int milliTime;
|
||||
@ -407,11 +410,11 @@ const char* g_stratName[] = { "ZSTD_fast ",
|
||||
"ZSTD_btlazy2",
|
||||
"ZSTD_btopt " };
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
|
||||
{
|
||||
DISPLAY("\r%79s\r", "");
|
||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
|
||||
0, params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength,
|
||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
|
||||
params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength,
|
||||
params.targetLength, g_stratName[(U32)(params.strategy)]);
|
||||
fprintf(f,
|
||||
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
|
||||
@ -423,7 +426,7 @@ static U32 g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0 }; /* NB_LEVELS_TRACKED : c
|
||||
|
||||
typedef struct {
|
||||
BMK_result_t result;
|
||||
ZSTD_parameters params;
|
||||
ZSTD_compressionParameters params;
|
||||
} winnerInfo_t;
|
||||
|
||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
@ -431,7 +434,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
|
||||
unsigned cLevel;
|
||||
|
||||
fprintf(f, "\n /* Proposed configurations : */ \n");
|
||||
fprintf(f, " /* l, W, C, H, S, L, T, strat */ \n");
|
||||
fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
|
||||
|
||||
for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++)
|
||||
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
|
||||
@ -446,9 +449,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
|
||||
BMK_printWinners2(stdout, winners, srcSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_sizeofCCtx(ZSTD_parameters params); /* hidden interface, declared here */
|
||||
size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters params); /* hidden interface, declared here */
|
||||
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
@ -541,7 +544,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
|
||||
|
||||
|
||||
/* nullified useless params, to ensure count stats */
|
||||
static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
|
||||
static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
|
||||
{
|
||||
g_params = params;
|
||||
if (params.strategy == ZSTD_fast)
|
||||
@ -552,44 +555,50 @@ static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
|
||||
}
|
||||
|
||||
|
||||
static void paramVariation(ZSTD_parameters* p)
|
||||
static void paramVariation(ZSTD_compressionParameters* ptr)
|
||||
{
|
||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
||||
for (; nbChanges; nbChanges--) {
|
||||
const U32 changeID = FUZ_rand(&g_rand) % 14;
|
||||
switch(changeID)
|
||||
{
|
||||
case 0:
|
||||
p->contentLog++; break;
|
||||
case 1:
|
||||
p->contentLog--; break;
|
||||
case 2:
|
||||
p->hashLog++; break;
|
||||
case 3:
|
||||
p->hashLog--; break;
|
||||
case 4:
|
||||
p->searchLog++; break;
|
||||
case 5:
|
||||
p->searchLog--; break;
|
||||
case 6:
|
||||
p->windowLog++; break;
|
||||
case 7:
|
||||
p->windowLog--; break;
|
||||
case 8:
|
||||
p->searchLength++; break;
|
||||
case 9:
|
||||
p->searchLength--; break;
|
||||
case 10:
|
||||
p->strategy = (ZSTD_strategy)(((U32)p->strategy)+1); break;
|
||||
case 11:
|
||||
p->strategy = (ZSTD_strategy)(((U32)p->strategy)-1); break;
|
||||
case 12:
|
||||
p->targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
case 13:
|
||||
p->targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
ZSTD_compressionParameters p;
|
||||
U32 validated = 0;
|
||||
while (!validated) {
|
||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
||||
p = *ptr;
|
||||
for ( ; nbChanges ; nbChanges--) {
|
||||
const U32 changeID = FUZ_rand(&g_rand) % 14;
|
||||
switch(changeID)
|
||||
{
|
||||
case 0:
|
||||
p.contentLog++; break;
|
||||
case 1:
|
||||
p.contentLog--; break;
|
||||
case 2:
|
||||
p.hashLog++; break;
|
||||
case 3:
|
||||
p.hashLog--; break;
|
||||
case 4:
|
||||
p.searchLog++; break;
|
||||
case 5:
|
||||
p.searchLog--; break;
|
||||
case 6:
|
||||
p.windowLog++; break;
|
||||
case 7:
|
||||
p.windowLog--; break;
|
||||
case 8:
|
||||
p.searchLength++; break;
|
||||
case 9:
|
||||
p.searchLength--; break;
|
||||
case 10:
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
|
||||
case 11:
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
|
||||
case 12:
|
||||
p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
case 13:
|
||||
p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
}
|
||||
}
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||
}
|
||||
ZSTD_validateParams(p);
|
||||
*ptr = p;
|
||||
}
|
||||
|
||||
|
||||
@ -605,7 +614,7 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||
|
||||
static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
ZSTD_parameters params,
|
||||
ZSTD_compressionParameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
@ -613,7 +622,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
const int startTime = BMK_GetMilliStart();
|
||||
|
||||
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) {
|
||||
ZSTD_parameters p = params;
|
||||
ZSTD_compressionParameters p = params;
|
||||
|
||||
if (nbVariations++ > g_maxNbVariations) break;
|
||||
paramVariation(&p);
|
||||
@ -634,10 +643,12 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
}
|
||||
|
||||
|
||||
static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance)
|
||||
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
|
||||
{
|
||||
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
|
||||
if (!chance) {
|
||||
U32 validated = 0;
|
||||
if (!chance)
|
||||
while (!validated) {
|
||||
/* totally random entry */
|
||||
p->contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
|
||||
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||
@ -646,7 +657,7 @@ static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance)
|
||||
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
||||
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
||||
ZSTD_validateParams(p);
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(*p));
|
||||
}
|
||||
}
|
||||
|
||||
@ -658,10 +669,9 @@ static void BMK_selectRandomStart(
|
||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
||||
/* totally random entry */
|
||||
ZSTD_parameters p;
|
||||
ZSTD_compressionParameters p;
|
||||
potentialRandomParams(&p, 1);
|
||||
p.srcSize = srcSize;
|
||||
ZSTD_validateParams(&p);
|
||||
ZSTD_adjustCParams(&p, srcSize, 0);
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
else
|
||||
@ -672,7 +682,7 @@ static void BMK_selectRandomStart(
|
||||
static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_parameters params;
|
||||
ZSTD_compressionParameters params;
|
||||
winnerInfo_t winners[NB_LEVELS_TRACKED];
|
||||
int i;
|
||||
unsigned u;
|
||||
@ -682,8 +692,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
|
||||
if (g_singleRun) {
|
||||
BMK_result_t testResult;
|
||||
g_params.srcSize = blockSize;
|
||||
ZSTD_validateParams(&g_params);
|
||||
ZSTD_adjustCParams(&g_params, srcSize, 0);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||
DISPLAY("\n");
|
||||
return;
|
||||
@ -699,7 +708,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
else {
|
||||
/* baseline config for level 1 */
|
||||
BMK_result_t testResult;
|
||||
params = ZSTD_getParams(1, blockSize);
|
||||
params = ZSTD_getCParams(1, blockSize, 0);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
|
||||
}
|
||||
@ -712,8 +721,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
for (i=1; i<=maxSeeds; i++) {
|
||||
params = ZSTD_getParams(i, blockSize);
|
||||
ZSTD_validateParams(¶ms);
|
||||
params = ZSTD_getCParams(i, blockSize, 0);
|
||||
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
}
|
||||
@ -864,7 +872,7 @@ int optimizeForSize(char* inFileName)
|
||||
|
||||
{
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_parameters params;
|
||||
ZSTD_compressionParameters params;
|
||||
winnerInfo_t winner;
|
||||
BMK_result_t candidate;
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
|
||||
@ -878,7 +886,7 @@ int optimizeForSize(char* inFileName)
|
||||
{
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
for (i=1; i<=maxSeeds; i++) {
|
||||
params = ZSTD_getParams(i, blockSize);
|
||||
params = ZSTD_getCParams(i, blockSize, 0);
|
||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
|
||||
if ( (candidate.cSize < winner.result.cSize)
|
||||
||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) )
|
||||
@ -1028,7 +1036,7 @@ int main(int argc, char** argv)
|
||||
case 'S':
|
||||
g_singleRun = 1;
|
||||
argument++;
|
||||
g_params = ZSTD_getParams(2, g_blockSize);
|
||||
g_params = ZSTD_getCParams(2, g_blockSize, 0);
|
||||
for ( ; ; ) {
|
||||
switch(*argument)
|
||||
{
|
||||
@ -1079,7 +1087,7 @@ int main(int argc, char** argv)
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
cLevel *= 10, cLevel += *argument++ - '0';
|
||||
g_params = ZSTD_getParams(cLevel, g_blockSize);
|
||||
g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
|
||||
continue;
|
||||
}
|
||||
default : ;
|
||||
|
@ -25,7 +25,9 @@ roundTripTest() {
|
||||
|
||||
echo "\n**** simple tests **** "
|
||||
./datagen > tmp
|
||||
$ZSTD tmp
|
||||
echo -n "trivial compression : "
|
||||
$ZSTD -f tmp
|
||||
echo "OK"
|
||||
$ZSTD -99 tmp && die "too large compression level undetected"
|
||||
$ZSTD tmp -c > tmpCompressed
|
||||
$ZSTD tmp --stdout > tmpCompressed
|
||||
@ -71,6 +73,11 @@ echo "\n**** dictionary tests **** "
|
||||
./datagen -g1M | md5sum > tmp1
|
||||
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
|
||||
diff -q tmp1 tmp2
|
||||
$ZSTD --train *.c *.h -o tmpDict
|
||||
$ZSTD xxhash.c -D tmpDict -of tmp
|
||||
$ZSTD -d tmp -D tmpDict -of result
|
||||
diff xxhash.c result
|
||||
|
||||
|
||||
echo "\n**** multiple files tests **** "
|
||||
|
||||
@ -106,6 +113,10 @@ $ZSTD -t * && die "bad files not detected !"
|
||||
echo "\n**** zstd round-trip tests **** "
|
||||
|
||||
roundTripTest
|
||||
roundTripTest -g15K # TableID==3
|
||||
roundTripTest -g127K # TableID==2
|
||||
roundTripTest -g255K # TableID==1
|
||||
roundTripTest -g513K # TableID==0
|
||||
roundTripTest -g512K 6 # greedy, hash chain
|
||||
roundTripTest -g512K 16 # btlazy2
|
||||
roundTripTest -g512K 19 # btopt
|
||||
|
@ -1,6 +1,6 @@
|
||||
/*
|
||||
Fuzzer test tool for zstd_buffered
|
||||
Copyright (C) Yann Collet 2105
|
||||
Copyright (C) Yann Collet 2015-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
@ -19,11 +19,10 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
- ZSTD homepage : https://www.zstd.net/
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
@ -33,7 +32,7 @@
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
@ -42,13 +41,13 @@
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#include "zbuff.h"
|
||||
#include "zstd.h" /* ZSTD_compressBound() */
|
||||
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
@ -66,7 +65,7 @@ static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Display Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
@ -83,7 +82,7 @@ static U32 g_displayTime = 0;
|
||||
static U32 g_testTime = 0;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
@ -107,15 +106,17 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
/*! FUZ_rand() :
|
||||
@return : a 27 bits random value, from a 32-bits `seed`.
|
||||
`seed` is also modified */
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* src)
|
||||
unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
U32 rand32 = *seedPtr;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
*seedPtr = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
@ -133,12 +134,12 @@ static unsigned FUZ_highbit32(U32 v32)
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
int testResult = 0;
|
||||
void* CNBuffer;
|
||||
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
void* compressedBuffer;
|
||||
size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* decodedBuffer;
|
||||
size_t decodedBufferSize = CNBufferSize;
|
||||
void* CNBuffer = malloc(CNBufferSize);
|
||||
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* compressedBuffer = malloc(compressedBufferSize);
|
||||
size_t const decodedBufferSize = CNBufferSize;
|
||||
void* decodedBuffer = malloc(decodedBufferSize);
|
||||
U32 randState = seed;
|
||||
size_t result, cSize, readSize, genSize;
|
||||
U32 testNb=0;
|
||||
@ -146,11 +147,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
|
||||
|
||||
/* Create compressible test buffer */
|
||||
CNBuffer = malloc(CNBufferSize);
|
||||
compressedBuffer = malloc(compressedBufferSize);
|
||||
decodedBuffer = malloc(decodedBufferSize);
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd)
|
||||
{
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
goto _output_error;
|
||||
}
|
||||
@ -183,11 +180,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
{
|
||||
size_t i;
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++)
|
||||
{
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
@ -213,8 +208,7 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t i;
|
||||
for (i=0; i<max; i++)
|
||||
{
|
||||
for (i=0; i<max; i++) {
|
||||
if (b1[i] != b2[i]) break;
|
||||
}
|
||||
return i;
|
||||
@ -225,28 +219,39 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
|
||||
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
|
||||
{
|
||||
size_t const lengthMask = ((size_t)1 << logLength) - 1;
|
||||
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
|
||||
}
|
||||
|
||||
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
{
|
||||
U32 const logLength = FUZ_rand(seed) % maxLog;
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
BYTE* cNoiseBuffer[5];
|
||||
BYTE* srcBuffer;
|
||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
BYTE* copyBuffer;
|
||||
size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
|
||||
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
|
||||
BYTE* cBuffer;
|
||||
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
|
||||
BYTE* dstBuffer;
|
||||
size_t dstBufferSize = srcBufferSize;
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
U32 coreSeed = seed;
|
||||
ZBUFF_CCtx* zc;
|
||||
ZBUFF_DCtx* zd;
|
||||
U32 startTime = FUZ_GetMilliStart();
|
||||
|
||||
/* allocation */
|
||||
/* allocations */
|
||||
zc = ZBUFF_createCCtx();
|
||||
zd = ZBUFF_createDCtx();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
@ -267,172 +272,144 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||
srcBuffer = cNoiseBuffer[2];
|
||||
memset(copyBuffer, 0x65, copyBufferSize);
|
||||
memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */
|
||||
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
|
||||
|
||||
/* catch up testNb */
|
||||
for (testNb=1; testNb < startTest; testNb++)
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
|
||||
{
|
||||
size_t sampleSize, sampleStart;
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
const BYTE* dict;
|
||||
size_t cSize, dictSize;
|
||||
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
|
||||
size_t maxTestSize, dictSize;
|
||||
size_t cSize, totalTestSize, totalCSize, totalGenSize;
|
||||
size_t errorCode;
|
||||
U32 sampleSizeLog, buffNb, n, nbChunks;
|
||||
U32 n, nbChunks;
|
||||
XXH64_CREATESTATE_STATIC(xxh64);
|
||||
U64 crcOrig, crcDest;
|
||||
U64 crcOrig;
|
||||
|
||||
/* init */
|
||||
DISPLAYUPDATE(2, "\r%6u", testNb);
|
||||
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime1;
|
||||
buffNb = FUZ_rand(&lseed) & 127;
|
||||
if (buffNb & 7) buffNb=2; /* select buffer */
|
||||
else
|
||||
{
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7)
|
||||
{
|
||||
const U32 tnb[2] = { 1, 3 };
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
}
|
||||
else
|
||||
{
|
||||
const U32 tnb[2] = { 0, 4 };
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
}
|
||||
}
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
|
||||
/* Multi - segments compression test */
|
||||
/* state total reset */
|
||||
/* some problems only happen when states are re-used in a specific order */
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
||||
|
||||
/* srcBuffer selection [0-4] */
|
||||
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
|
||||
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
|
||||
else {
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7) {
|
||||
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} else {
|
||||
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} }
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
}
|
||||
|
||||
/* compression init */
|
||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
/* random dictionary selection */
|
||||
{ size_t dictStart;
|
||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
}
|
||||
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
|
||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||
} }
|
||||
|
||||
/* multi-segments compression test */
|
||||
XXH64_reset(xxh64, 0);
|
||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
maxTestSize = (size_t)1 << sampleSizeLog;
|
||||
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
|
||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
|
||||
/* compress random chunk into random size dst buffer */
|
||||
{ size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
|
||||
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
dict = srcBuffer + sampleStart;
|
||||
dictSize = sampleSize;
|
||||
ZBUFF_compressInitDictionary(zc, dict, dictSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
|
||||
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
|
||||
CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError));
|
||||
|
||||
totalTestSize = 0;
|
||||
cSize = 0;
|
||||
for (n=0; n<nbChunks; n++)
|
||||
{
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
readSize = sampleSize;
|
||||
|
||||
/* random size output buffer */
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
genSize = MIN (cBufferSize - cSize, sampleSize);
|
||||
|
||||
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
|
||||
XXH64_update(xxh64, srcBuffer+sampleStart, readSize);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
|
||||
cSize += genSize;
|
||||
totalTestSize += readSize;
|
||||
|
||||
if ((FUZ_rand(&lseed) & 15) == 0)
|
||||
{
|
||||
/* add a few random flushes operations, to mess around */
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
genSize = MIN (cBufferSize - cSize, sampleSize);
|
||||
errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
|
||||
cSize += genSize;
|
||||
XXH64_update(xxh64, srcBuffer+srcStart, readChunkSize);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
|
||||
cSize += dstBuffSize;
|
||||
totalTestSize += readChunkSize;
|
||||
}
|
||||
|
||||
if (totalTestSize > maxTestSize) break;
|
||||
/* random flush operation, to mess around */
|
||||
if ((FUZ_rand(&lseed) & 15) == 0) {
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
|
||||
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
|
||||
cSize += dstBuffSize;
|
||||
} }
|
||||
|
||||
/* final frame epilogue */
|
||||
{ size_t dstBuffSize = cBufferSize - cSize;
|
||||
size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
|
||||
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
|
||||
cSize += dstBuffSize;
|
||||
}
|
||||
genSize = cBufferSize - cSize;
|
||||
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
CHECK (errorCode != 0, "frame epilogue not fully consumed");
|
||||
cSize += genSize;
|
||||
crcOrig = XXH64_digest(xxh64);
|
||||
|
||||
/* multi - fragments decompression test */
|
||||
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while (totalCSize < cSize)
|
||||
{
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
readSize = sampleSize;
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
|
||||
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
totalGenSize += genSize;
|
||||
totalCSize += readSize;
|
||||
for (totalCSize = 0, totalGenSize = 0 ; totalCSize < cSize ; ) {
|
||||
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
|
||||
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
|
||||
CHECK (ZBUFF_isError(decompressError), "decompression error : %s", ZBUFF_getErrorName(decompressError));
|
||||
totalGenSize += dstBuffSize;
|
||||
totalCSize += readCSrcSize;
|
||||
errorCode = decompressError; /* needed for != 0 last test */
|
||||
}
|
||||
CHECK (errorCode != 0, "frame not fully decoded");
|
||||
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
|
||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||
crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
|
||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (crcDest!=crcOrig, "decompressed data corrupted"); }
|
||||
|
||||
/*===== noisy/erroneous src decompression test =====*/
|
||||
|
||||
/* noisy/erroneous src decompression test */
|
||||
/* add some noise */
|
||||
nbChunks = (FUZ_rand(&lseed) & 7) + 2;
|
||||
for (n=0; n<nbChunks; n++)
|
||||
{
|
||||
size_t cStart;
|
||||
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
if (sampleSize > cSize/3) sampleSize = cSize/3;
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
cStart = FUZ_rand(&lseed) % (cSize - sampleSize);
|
||||
|
||||
memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
|
||||
}
|
||||
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
|
||||
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
|
||||
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
|
||||
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
|
||||
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
|
||||
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
|
||||
} }
|
||||
|
||||
/* try decompression on noisy data */
|
||||
ZBUFF_decompressInit(zd);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) )
|
||||
{
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
readSize = sampleSize;
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
|
||||
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
|
||||
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
|
||||
totalGenSize += genSize;
|
||||
totalCSize += readSize;
|
||||
}
|
||||
}
|
||||
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
|
||||
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
|
||||
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
|
||||
if (ZBUFF_isError(decompressError)) break; /* error correctly detected */
|
||||
totalGenSize += dstBuffSize;
|
||||
totalCSize += readCSrcSize;
|
||||
} }
|
||||
DISPLAY("\r%u fuzzer tests completed \n", testNb);
|
||||
|
||||
_cleanup:
|
||||
@ -454,10 +431,10 @@ _output_error:
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(char* programName)
|
||||
int FUZ_usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
@ -474,7 +451,7 @@ int FUZ_usage(char* programName)
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
@ -484,23 +461,18 @@ int main(int argc, char** argv)
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
U32 mainPause = 0;
|
||||
char* programName;
|
||||
const char* programName = argv[0];
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
/* Parsing commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
while (*argument!=0)
|
||||
{
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
@ -521,8 +493,7 @@ int main(int argc, char** argv)
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
@ -532,8 +503,7 @@ int main(int argc, char** argv)
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
g_testTime *= 10;
|
||||
g_testTime += *argument - '0';
|
||||
argument++;
|
||||
@ -547,8 +517,7 @@ int main(int argc, char** argv)
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
@ -558,8 +527,7 @@ int main(int argc, char** argv)
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
@ -569,8 +537,7 @@ int main(int argc, char** argv)
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
@ -582,9 +549,7 @@ int main(int argc, char** argv)
|
||||
default:
|
||||
return FUZ_usage(programName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
|
||||
@ -598,8 +563,8 @@ int main(int argc, char** argv)
|
||||
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
if (mainPause)
|
||||
{
|
||||
|
||||
if (mainPause) {
|
||||
int unused;
|
||||
DISPLAY("Press Enter \n");
|
||||
unused = getchar();
|
||||
|
Loading…
Reference in New Issue
Block a user