AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zbuff supports byte-by-byte decompression scenarios

Browse Source
This commit is contained in:
Yann Collet 2016-05-10 14:14:19 +02:00
parent 918b934a26
commit bd39d54576
4 changed files with 81 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
lib/decompress/zbuff_decompress.c
View File

@ -64,14 +64,14 @@
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */
struct ZBUFF_DCtx_s {
ZSTD_DCtx* zd;
ZSTD_frameParams fParams;
size_t blockSize;
ZBUFF_dStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s {
size_t outBuffSize;
size_t outStart;
size_t outEnd;
ZBUFF_dStage stage;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
{
zbd->stage = ZBUFFds_readHeader;
zbd->inPos = zbd->outStart = zbd->outEnd = 0;
zbd->stage = ZBUFFds_loadHeader;
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
}
@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
case ZBUFFds_init :
return ERROR(init_missing);
case ZBUFFds_readHeader :
/* read header from src */
{ size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) {
/* not enough input to decode header : needs headerSize > *srcSizePtr */
*dstCapacityPtr = 0;
*srcSizePtr = 0;
return headerSize;
case ZBUFFds_loadHeader :
{ size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
if (hSize != 0) {
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
if (ZSTD_isError(hSize)) return hSize;
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
zbd->lhSize += iend-ip; ip = iend; notDone = 0;
*dstCapacityPtr = 0;
return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
if (ZSTD_isError(h1Result)) return h1Result;
if (h1Size < zbd->lhSize) { /* long header */
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} }
/* Frame header instruct buffer sizes */
@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
notDone = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) {
/* directly decode from src */
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
ip, neededInSize);
@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
ip += loadedSize;
zbd->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,

42
lib/decompress/zstd_decompress.c
View File

@ -657,7 +657,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
}
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
//FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@ -964,31 +965,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
return dctx->expected;
}
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
/* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
ZSTD_checkContinuity(dctx, dst);
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
/* Decompress : frame header; part 1 */
switch (dctx->stage)
{
case ZSTDds_getFrameHeaderSize :
{
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
dctx->stage = ZSTDds_decodeFrameHeader;
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
dctx->stage = ZSTDds_decodeFrameHeader;
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
case ZSTDds_decodeFrameHeader:
{
size_t result;
{ size_t result;
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
if (ZSTD_isError(result)) return result;
@ -997,8 +996,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
return 0;
}
case ZSTDds_decodeBlockHeader:
{
blockProperties_t bp;
{ blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (bp.blockType == bt_end) {
@ -1012,16 +1010,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
return 0;
}
case ZSTDds_decompressBlock:
{
/* Decompress : block content */
size_t rSize;
{ size_t rSize;
switch(dctx->bType)
{
case bt_compressed:
rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize);
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
break;
case bt_raw :
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet handled */

6
programs/fileio.c
View File

@ -533,9 +533,9 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); /* can be <= toLoad (null string) */
/* Header loading (optional, saves one loop) */
{ size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
readSize = alreadyLoaded + loadedSize;
}

27
programs/zbufftest.c
View File

@ -188,6 +188,33 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
}
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
{ size_t r = 1, pIn=0, pOut=0;
while (r) {
size_t inS = 1;
size_t outS = 1;
r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
pIn += inS;
pOut += outS;
}
readSize = pIn;
genSize = pOut;
}
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
{ size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
_end:
ZBUFF_freeCCtx(zc);
ZBUFF_freeDCtx(zd);