From bd39d54576f50724a08eaa8e3ee864c9f2c5d5b1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 10 May 2016 14:14:19 +0200 Subject: [PATCH] zbuff supports byte-by-byte decompression scenarios --- lib/decompress/zbuff_decompress.c | 48 ++++++++++++++++++++----------- lib/decompress/zstd_decompress.c | 42 ++++++++++++--------------- programs/fileio.c | 6 ++-- programs/zbufftest.c | 27 +++++++++++++++++ 4 files changed, 81 insertions(+), 42 deletions(-) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index faa81370..858b442f 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -64,14 +64,14 @@ * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ -typedef enum { ZBUFFds_init, ZBUFFds_readHeader, +typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; /* *** Resource management *** */ struct ZBUFF_DCtx_s { ZSTD_DCtx* zd; ZSTD_frameParams fParams; - size_t blockSize; + ZBUFF_dStage stage; char* inBuff; size_t inBuffSize; size_t inPos; @@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s { size_t outBuffSize; size_t outStart; size_t outEnd; - ZBUFF_dStage stage; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) { - zbd->stage = ZBUFFds_readHeader; - zbd->inPos = zbd->outStart = zbd->outEnd = 0; + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); } @@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, case ZBUFFds_init : return ERROR(init_missing); - case ZBUFFds_readHeader : - /* read header from src */ - { size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr); - if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) { - /* not enough input to decode header : needs headerSize > *srcSizePtr */ - *dstCapacityPtr = 0; - *srcSizePtr = 0; - return headerSize; + case ZBUFFds_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (hSize != 0) { + size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ + if (ZSTD_isError(hSize)) return hSize; + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + zbd->lhSize += iend-ip; ip = iend; notDone = 0; + *dstCapacityPtr = 0; + return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; + if (h1Size < zbd->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; } } /* Frame header instruct buffer sizes */ @@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, notDone = 0; break; } - if ((size_t)(iend-ip) >= neededInSize) { - /* directly decode from src */ + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, ip, neededInSize); @@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, ip += loadedSize; zbd->inPos += loadedSize; if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ + /* decode loaded input */ { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 72e52a45..7263d29b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -657,7 +657,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) } -FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, +//FORCE_INLINE +size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) @@ -964,31 +965,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } -size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ if (srcSize != dctx->expected) return ERROR(srcSize_wrong); - ZSTD_checkContinuity(dctx, dst); + if (dstCapacity) ZSTD_checkContinuity(dctx, dst); /* Decompress : frame header; part 1 */ switch (dctx->stage) { case ZSTDds_getFrameHeaderSize : - { - if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); - if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (dctx->headerSize > ZSTD_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + if (dctx->headerSize > ZSTD_frameHeaderSize_min) { + dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; } + dctx->expected = 0; /* not necessary to copy more */ + case ZSTDds_decodeFrameHeader: - { - size_t result; + { size_t result; memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; @@ -997,8 +996,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co return 0; } case ZSTDds_decodeBlockHeader: - { - blockProperties_t bp; + { blockProperties_t bp; size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; if (bp.blockType == bt_end) { @@ -1012,16 +1010,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co return 0; } case ZSTDds_decompressBlock: - { - /* Decompress : block content */ - size_t rSize; + { size_t rSize; switch(dctx->bType) { case bt_compressed: - rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break; case bt_raw : - rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; case bt_rle : return ERROR(GENERIC); /* not yet handled */ diff --git a/programs/fileio.c b/programs/fileio.c index 36b1ff29..dea08690 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -533,9 +533,9 @@ unsigned long long FIO_decompressFrame(dRess_t ress, ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); - /* Complete Header loading */ - { size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */ - size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); /* can be <= toLoad (null string) */ + /* Header loading (optional, saves one loop) */ + { size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */ + size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); readSize = alreadyLoaded + loadedSize; } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f44f5d5e..f1fa842c 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -188,6 +188,33 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); } + /* Byte-by-byte decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + { size_t r = 1, pIn=0, pOut=0; + while (r) { + size_t inS = 1; + size_t outS = 1; + r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); + pIn += inS; + pOut += outS; + } + readSize = pIn; + genSize = pOut; + } + if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + { size_t i; + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + for (i=0; i