Integrated huff0 (breaking format change)
This commit is contained in:
parent
8b48b24821
commit
e8c6bb1e42
30
lib/fse.h
30
lib/fse.h
@ -70,7 +70,33 @@ FSE_decompress():
|
||||
|
||||
** Important ** : FSE_decompress() doesn't decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
FSE library doesn't manage headers, which are intentionally left to the user layer.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* Huff0 simple functions
|
||||
******************************************/
|
||||
size_t HUF_compress (void* dst, size_t dstSize, const void* src, size_t srcSize);
|
||||
size_t HUF_decompress(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
/*
|
||||
HUF_compress():
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated, and sized to handle worst case situations.
|
||||
Worst case size evaluation is provided by FSE_compressBound().
|
||||
return : size of compressed data
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), it's an error code.
|
||||
|
||||
HUF_decompress():
|
||||
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'maxDstSize'.
|
||||
return : size of regenerated data (<= maxDstSize)
|
||||
or an error code, which can be tested using FSE_isError()
|
||||
|
||||
** Important ** : HUF_decompress() doesn't decompress non-compressible nor RLE data !!!
|
||||
*/
|
||||
|
||||
|
||||
@ -98,6 +124,8 @@ FSE_compress2():
|
||||
*/
|
||||
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE detailed API
|
||||
|
@ -50,10 +50,16 @@ extern "C" {
|
||||
******************************************/
|
||||
#define FSE_MAX_HEADERSIZE 512
|
||||
#define FSE_COMPRESSBOUND(size) (size + (size>>7) + FSE_MAX_HEADERSIZE) /* Macro can be useful for static allocation */
|
||||
/* You can statically allocate a CTable as a table of unsigned using below macro */
|
||||
|
||||
/* You can statically allocate CTable/DTable as a table of unsigned using below macro */
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
/* You can statically allocate a Huff0 DTable as a table of unsigned char using below macro */
|
||||
#define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
|
||||
|
||||
|
||||
/******************************************
|
||||
* Error Management
|
||||
@ -194,6 +200,12 @@ unsigned int FSE_reloadDStream(FSE_DStream_t* bitD);
|
||||
unsigned FSE_endOfDStream(const FSE_DStream_t* bitD);
|
||||
unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
typedef enum { FSE_DStream_unfinished = 0,
|
||||
FSE_DStream_partiallyFilled = 1,
|
||||
FSE_DStream_completed = 2,
|
||||
FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
|
||||
|
||||
/*
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
@ -218,23 +230,23 @@ Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25
|
||||
unsigned int bitField = FSE_readBits(&DStream, nbBits);
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = FSE_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size is controlled by bitD_t==32 bits).
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
FSE_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
0 : there is still some data left into the DStream.
|
||||
1 : Dstream reached end of buffer, but is not yet fully extracted. It will not load data from memory any more.
|
||||
2 : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
3 : Dstream went too far. Decompression result is corrupted.
|
||||
FSE_DStream_unfinished : there is still some data left into the DStream.
|
||||
FSE_DStream_partiallyFilled : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer(1), progress slowly, notably if you decode multiple symbols per loop,
|
||||
When reaching end of buffer (FSE_DStream_partiallyFilled), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
FSE_reloadDStream(&DStream) >= 2
|
||||
FSE_reloadDStream(&DStream) >= FSE_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
|
76
lib/zstd.c
76
lib/zstd.c
@ -124,7 +124,7 @@ typedef unsigned long long U64;
|
||||
/********************************************************
|
||||
* Constants
|
||||
*********************************************************/
|
||||
static const U32 ZSTD_magicNumber = 0xFD2FB51C; /* Initial (limited) frame format */
|
||||
static const U32 ZSTD_magicNumber = 0xFD2FB51D; /* 2nd magic number (huff0) */
|
||||
|
||||
#define HASH_LOG (ZSTD_MEMORY_USAGE - 2)
|
||||
#define HASH_TABLESIZE (1 << HASH_LOG)
|
||||
@ -512,7 +512,7 @@ static size_t ZSTD_compressRle (void* dst, size_t maxDstSize, const void* src, s
|
||||
/* Build header */
|
||||
ostart[0] = (BYTE)(srcSize>>16);
|
||||
ostart[1] = (BYTE)(srcSize>>8);
|
||||
ostart[2] = (BYTE)srcSize;
|
||||
ostart[2] = (BYTE) srcSize;
|
||||
ostart[0] += (BYTE)(bt_rle<<6);
|
||||
|
||||
return ZSTD_blockHeaderSize+1;
|
||||
@ -527,9 +527,9 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* sr
|
||||
memcpy(ostart + ZSTD_blockHeaderSize, src, srcSize);
|
||||
|
||||
/* Build header */
|
||||
ostart[0] = (BYTE)(srcSize>>16);
|
||||
ostart[1] = (BYTE)(srcSize>>8);
|
||||
ostart[2] = (BYTE)srcSize;
|
||||
ostart[0] = (BYTE)(srcSize>>16);
|
||||
ostart[1] = (BYTE)(srcSize>>8);
|
||||
ostart[2] = (BYTE) srcSize;
|
||||
ostart[0] += (BYTE)(bt_raw<<6); /* is a raw (uncompressed) block */
|
||||
|
||||
return ZSTD_blockHeaderSize+srcSize;
|
||||
@ -537,7 +537,7 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* sr
|
||||
|
||||
|
||||
/* return : size of CStream in bits */
|
||||
static size_t ZSTD_compressLiterals_usingCTable(void* dst, size_t dstSize,
|
||||
size_t ZSTD_compressLiterals_usingCTable(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const FSE_CTable* CTable)
|
||||
{
|
||||
@ -603,6 +603,33 @@ size_t ZSTD_minGain(size_t srcSize)
|
||||
static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const size_t minGain = ZSTD_minGain(srcSize);
|
||||
|
||||
#if 1
|
||||
#define LHSIZE 5
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
size_t hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
|
||||
if (hsize<2) return hsize; /* special cases */
|
||||
if (hsize >= srcSize - minGain) return 0;
|
||||
|
||||
hsize += 2; /* work around vs fixed 3-bytes header */
|
||||
|
||||
/* Build header */
|
||||
{
|
||||
ostart[0] = (BYTE)(bt_compressed<<6); /* is a block, is compressed */
|
||||
ostart[0] += (BYTE)(hsize>>16);
|
||||
ostart[1] = (BYTE)(hsize>>8);
|
||||
ostart[2] = (BYTE)(hsize>>0);
|
||||
ostart[0] += (BYTE)((srcSize>>16)<<3);
|
||||
ostart[3] = (BYTE)(srcSize>>8);
|
||||
ostart[4] = (BYTE)(srcSize>>0);
|
||||
}
|
||||
|
||||
hsize -= 2;
|
||||
return hsize+LHSIZE;
|
||||
|
||||
#else
|
||||
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
const BYTE* ip = istart;
|
||||
|
||||
@ -616,7 +643,6 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
|
||||
S16 norm[256];
|
||||
U32 CTable[ FSE_CTABLE_SIZE_U32(LitFSELog, 256) ];
|
||||
size_t errorCode;
|
||||
const size_t minGain = ZSTD_minGain(srcSize);
|
||||
|
||||
/* early out */
|
||||
if (dstSize < FSE_compressBound(srcSize)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
@ -658,6 +684,8 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
|
||||
#endif // 1
|
||||
}
|
||||
|
||||
|
||||
@ -698,6 +726,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
{
|
||||
size_t cSize;
|
||||
size_t litSize = op_lit - op_lit_start;
|
||||
|
||||
if (litSize <= LITERAL_NOENTROPY) cSize = ZSTD_noCompressBlock (op, maxDstSize, op_lit_start, litSize);
|
||||
else
|
||||
{
|
||||
@ -1304,7 +1333,7 @@ FORCE_INLINE size_t ZSTD_decompressLiterals_usingDTable_generic(
|
||||
return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompressLiterals_usingDTable(
|
||||
size_t ZSTD_decompressLiterals_usingDTable(
|
||||
void* const dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const FSE_DTable* DTable, U32 fast)
|
||||
@ -1313,9 +1342,27 @@ static size_t ZSTD_decompressLiterals_usingDTable(
|
||||
return ZSTD_decompressLiterals_usingDTable_generic(dst, maxDstSize, src, srcSize, DTable, 0);
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompressLiterals(void* ctx, void* dst, size_t maxDstSize,
|
||||
static size_t ZSTD_decompressLiterals(void* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
#if 1
|
||||
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + maxDstSize;
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t errorCode;
|
||||
size_t litSize = ip[1] + (ip[0]<<8);
|
||||
litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
|
||||
op = oend - litSize;
|
||||
|
||||
(void)ctx;
|
||||
errorCode = HUF_decompress(op, litSize, ip+2, srcSize-2);
|
||||
if (FSE_isError(errorCode))
|
||||
return errorCode;
|
||||
return litSize;
|
||||
|
||||
#else
|
||||
/* assumed : blockType == blockCompressed */
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
short norm[256];
|
||||
@ -1337,6 +1384,7 @@ static size_t ZSTD_decompressLiterals(void* ctx, void* dst, size_t maxDstSize,
|
||||
fastMode = (U32)errorCode;
|
||||
|
||||
return ZSTD_decompressLiterals_usingDTable (dst, maxDstSize, ip, srcSize, DTable, fastMode);
|
||||
#endif // 1
|
||||
}
|
||||
|
||||
|
||||
@ -1491,8 +1539,8 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize
|
||||
const BYTE* dumps;
|
||||
const BYTE* litPtr;
|
||||
const BYTE* litEnd;
|
||||
const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
const size_t dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
|
||||
const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
|
||||
FSE_DTable* DTableML = (FSE_DTable*)ctx;
|
||||
FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog);
|
||||
FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog);
|
||||
@ -1516,7 +1564,7 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize
|
||||
litEnd = ip - lastLLSize;
|
||||
ip += errorCode;
|
||||
|
||||
/* decompression */
|
||||
/* LZ decompression */
|
||||
{
|
||||
FSE_DStream_t DStream;
|
||||
FSE_DState_t stateLL, stateOffb, stateML;
|
||||
@ -1600,7 +1648,7 @@ _another_round:
|
||||
|
||||
if (offset < 8)
|
||||
{
|
||||
const size_t dec64 = dec64table[offset];
|
||||
const int dec64 = dec64table[offset];
|
||||
op[0] = match[0];
|
||||
op[1] = match[1];
|
||||
op[2] = match[2];
|
||||
@ -1670,7 +1718,7 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (ip+blockSize > iend)
|
||||
if (blockSize > remainingSize)
|
||||
return (size_t)-ZSTD_ERROR_wrongSrcSize;
|
||||
|
||||
switch(blockProperties.blockType)
|
||||
|
@ -297,34 +297,37 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, char* fileName, int cLe
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while (BMK_GetMilliStart() == milliTime);
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
|
||||
{
|
||||
ZSTD_decompress(resultBuffer, srcSize, compressedBuffer, cSize);
|
||||
nbLoops++;
|
||||
size_t result = ZSTD_decompress(resultBuffer, srcSize, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result))
|
||||
{
|
||||
DISPLAY("\n!!! Decompression error !!! %s !\n", ZSTD_getErrorName(result));
|
||||
break;
|
||||
}
|
||||
}
|
||||
milliTime = BMK_GetMilliSpan(milliTime);
|
||||
|
||||
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
|
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
|
||||
#endif
|
||||
|
||||
/* CRC Checking */
|
||||
crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck)
|
||||
{
|
||||
unsigned i = 0;
|
||||
unsigned i;
|
||||
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
while (i<srcSize)
|
||||
for (i=0; i<srcSize; i++)
|
||||
{
|
||||
if (((BYTE*)srcBuffer)[i] != ((BYTE*)resultBuffer)[i])
|
||||
{
|
||||
printf("\nDecoding error at pos %u \n", i);
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if (crcOrig == crcCheck)
|
||||
|
Loading…
Reference in New Issue
Block a user