AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

created entropy structure type

Browse Source
This commit is contained in:
Yann Collet 2017-02-25 18:33:31 -08:00
parent 8dff956dbf
commit 8629f0e41f
1 changed files with 44 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
lib/decompress/zstd_decompress.c
View File

@ -87,22 +87,26 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef struct {
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyTables_t;
struct ZSTD_DCtx_s
{
const FSE_DTable* LLTptr;
const FSE_DTable* MLTptr;
const FSE_DTable* OFTptr;
const HUF_DTable* HUFptr;
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
ZSTD_entropyTables_t entropy;
const void* previousDstEnd; /* detect continuity */
const void* base; /* start of current segment */
const void* vBase; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
U32 rep[ZSTD_REP_NUM];
ZSTD_frameParams fParams;
blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage;
@ -131,15 +135,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->base = NULL;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
dctx->litEntropy = dctx->fseEntropy = 0;
dctx->dictID = 0;
MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
dctx->LLTptr = dctx->LLTable;
dctx->MLTptr = dctx->MLTable;
dctx->OFTptr = dctx->OFTable;
dctx->HUFptr = dctx->hufTable;
MEM_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
dctx->LLTptr = dctx->entropy.LLTable;
dctx->MLTptr = dctx->entropy.MLTable;
dctx->OFTptr = dctx->entropy.OFTable;
dctx->HUFptr = dctx->entropy.hufTable;
return 0;
}
@ -186,13 +190,13 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
dstDCtx->dictID = srcDCtx->dictID;
dstDCtx->litEntropy = srcDCtx->litEntropy;
dstDCtx->fseEntropy = srcDCtx->fseEntropy;
dstDCtx->LLTptr = srcDCtx->LLTable;
dstDCtx->MLTptr = srcDCtx->MLTable;
dstDCtx->OFTptr = srcDCtx->OFTable;
dstDCtx->HUFptr = srcDCtx->hufTable;
dstDCtx->rep[0] = srcDCtx->rep[0];
dstDCtx->rep[1] = srcDCtx->rep[1];
dstDCtx->rep[2] = srcDCtx->rep[2];
dstDCtx->LLTptr = srcDCtx->entropy.LLTable;
dstDCtx->MLTptr = srcDCtx->entropy.MLTable;
dstDCtx->OFTptr = srcDCtx->entropy.OFTable;
dstDCtx->HUFptr = srcDCtx->entropy.hufTable;
dstDCtx->entropy.rep[0] = srcDCtx->entropy.rep[0];
dstDCtx->entropy.rep[1] = srcDCtx->entropy.rep[1];
dstDCtx->entropy.rep[2] = srcDCtx->entropy.rep[2];
}
}
@ -506,14 +510,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
( singleStream ?
HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
HUF_decompress4X_hufOnly (dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
@ -830,19 +834,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
ip++;
/* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr,
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
LLtype, MaxLL, LLFSELog,
ip, iend-ip, LL_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
ip += llhSize;
}
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr,
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
OFtype, MaxOff, OffFSELog,
ip, iend-ip, OF_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
ip += ofhSize;
}
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr,
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
MLtype, MaxML, MLFSELog,
ip, iend-ip, ML_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
@ -1104,7 +1108,7 @@ static size_t ZSTD_decompressSequences(
if (nbSeq) {
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@ -1121,7 +1125,7 @@ static size_t ZSTD_decompressSequences(
/* check if reached exact end */
if (nbSeq) return ERROR(corruption_detected);
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
@ -1330,7 +1334,7 @@ static size_t ZSTD_decompressSequencesLong(
seqState_t seqState;
int seqNb;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
seqState.base = base;
seqState.pos = (size_t)(op-base);
seqState.gotoDict = (iPtrDiff)(dictEnd - base);
@ -1365,7 +1369,7 @@ static size_t ZSTD_decompressSequencesLong(
}
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
@ -1843,7 +1847,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
if (dictSize <= 8) return ERROR(dictionary_corrupted);
dictPtr += 8; /* skip header = magic + dictID */
{ size_t const hSize = HUF_readDTableX4(dctx->hufTable, dictPtr, dictEnd-dictPtr);
{ size_t const hSize = HUF_readDTableX4(dctx->entropy.hufTable, dictPtr, dictEnd-dictPtr);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dictPtr += hSize;
}
@ -1853,7 +1857,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->entropy.OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
@ -1862,7 +1866,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->entropy.MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
@ -1871,17 +1875,18 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->entropy.LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
{ int i;
for (i=0; i<3; i++) {
U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
if (rep==0 || rep >= dictSize) return ERROR(dictionary_corrupted);
dctx->entropy.rep[i] = rep;
} }
dctx->litEntropy = dctx->fseEntropy = 1;
return dictPtr - (const BYTE*)dict;
}
@ -1900,6 +1905,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
dict = (const char*)dict + eSize;
dictSize -= eSize;
}
dctx->litEntropy = dctx->fseEntropy = 1;
/* reference dictionary content */
return ZSTD_refDictContent(dctx, dict, dictSize);