created ZSTD_frameParams
This commit is contained in:
Yann Collet
parent
27d3dad751
commit
0e491c01fe
53
lib/zbuff.c
53
lib/zbuff.c
@ -298,28 +298,29 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
|
||||
}
|
||||
|
||||
|
||||
|
||||
/** ************************************************
|
||||
* Streaming decompression
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operation.
|
||||
* A ZBUFF_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation.
|
||||
* ZBUFF_DCtx objects can be reused multiple times.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFF_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *maxDstSizePtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
|
||||
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or 0 when a frame is completely decoded
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory)
|
||||
* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
|
||||
* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* **************************************************/
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
|
||||
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
|
||||
@ -329,7 +330,7 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_dec
|
||||
#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
|
||||
struct ZBUFF_DCtx_s {
|
||||
ZSTD_DCtx* zc;
|
||||
ZSTD_parameters params;
|
||||
ZSTD_frameParams fParams;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
@ -381,14 +382,16 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
|
||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
const char* const istart = (const char*)src;
|
||||
const char* ip = istart;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
char* const ostart = (char*)dst;
|
||||
char* op = ostart;
|
||||
char* const oend = ostart + *maxDstSizePtr;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
U32 notDone = 1;
|
||||
|
||||
while (notDone) {
|
||||
@ -400,13 +403,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
|
||||
case ZBUFFds_readHeader :
|
||||
/* read header from src */
|
||||
{
|
||||
size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
|
||||
size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
|
||||
if (ZSTD_isError(headerSize)) return headerSize;
|
||||
if (headerSize) {
|
||||
/* not enough input to decode header : tell how many bytes would be necessary */
|
||||
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
|
||||
zbc->hPos += *srcSizePtr;
|
||||
*maxDstSizePtr = 0;
|
||||
*dstCapacityPtr = 0;
|
||||
zbc->stage = ZBUFFds_loadHeader;
|
||||
return headerSize - zbc->hPos;
|
||||
}
|
||||
@ -422,11 +425,11 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
|
||||
src, *srcSizePtr);
|
||||
zbc->hPos += headerSize;
|
||||
ip += headerSize;
|
||||
headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos);
|
||||
headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos);
|
||||
if (ZSTD_isError(headerSize)) return headerSize;
|
||||
if (headerSize) {
|
||||
/* not enough input to decode header : tell how many bytes would be necessary */
|
||||
*maxDstSizePtr = 0;
|
||||
*dstCapacityPtr = 0;
|
||||
return headerSize - zbc->hPos;
|
||||
}
|
||||
// zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
|
||||
@ -435,7 +438,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
|
||||
case ZBUFFds_decodeHeader:
|
||||
/* apply header to create / resize buffers */
|
||||
{
|
||||
size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
|
||||
size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog;
|
||||
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
|
||||
if (zbc->inBuffSize < neededInSize) {
|
||||
free(zbc->inBuff);
|
||||
@ -524,7 +527,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
|
||||
} }
|
||||
|
||||
*srcSizePtr = ip-istart;
|
||||
*maxDstSizePtr = op-ostart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
|
||||
{
|
||||
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
|
||||
17
lib/zbuff.h
17
lib/zbuff.h
@ -75,7 +75,7 @@ ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstC
|
||||
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
||||
|
||||
/*-*************************************************
|
||||
* Streaming compression
|
||||
* Streaming compression - howto
|
||||
*
|
||||
* A ZBUFF_CCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
||||
@ -127,26 +127,27 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
|
||||
const void* src, size_t* srcSizePtr);
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFF_DCtx objects can be reused multiple times.
|
||||
* Note that ZBUFF_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency)
|
||||
* or 0 when a frame is completely decoded
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize()
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
|
||||
|
||||
@ -103,7 +103,7 @@ struct ZSTD_CCtx_s
|
||||
size_t workSpaceSize;
|
||||
size_t blockSize;
|
||||
size_t hbSize;
|
||||
char headerBuffer[ZSTD_frameHeaderSize_max];
|
||||
char headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
|
||||
seqStore_t seqStore; /* sequences storage ptrs */
|
||||
U32* hashTable;
|
||||
|
||||
@ -136,7 +136,7 @@ struct ZSTD_DCtx_s
|
||||
const void* dictEnd;
|
||||
size_t expected;
|
||||
size_t headerSize;
|
||||
ZSTD_parameters params;
|
||||
ZSTD_frameParams fParams;
|
||||
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||
ZSTD_dStage stage;
|
||||
U32 flagStaticTables;
|
||||
@ -144,7 +144,7 @@ struct ZSTD_DCtx_s
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
|
||||
|
||||
size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }
|
||||
@ -159,7 +159,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
||||
dctx->dictEnd = NULL;
|
||||
dctx->hufTableX4[0] = HufLog;
|
||||
dctx->flagStaticTables = 0;
|
||||
dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */
|
||||
dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength);
|
||||
return 0;
|
||||
}
|
||||
@ -286,15 +286,15 @@ static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
|
||||
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 magicNumber;
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max;
|
||||
magicNumber = MEM_readLE32(src);
|
||||
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
memset(params, 0, sizeof(*params));
|
||||
params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
params->searchLength = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH;
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
fparamsPtr->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
fparamsPtr->mml = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH;
|
||||
if ((((const BYTE*)src)[4] >> 5) != 0) return ERROR(frameParameter_unsupported); /* reserved 3 bits */
|
||||
return 0;
|
||||
}
|
||||
@ -308,8 +308,8 @@ static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_
|
||||
size_t result;
|
||||
if (srcSize != zc->headerSize)
|
||||
return ERROR(srcSize_wrong);
|
||||
result = ZSTD_getFrameParams(&(zc->params), src, srcSize);
|
||||
if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
|
||||
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -792,7 +792,7 @@ static size_t ZSTD_decompressSequences(
|
||||
const BYTE* const base = (const BYTE*) (dctx->base);
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
const U32 mls = dctx->params.searchLength;
|
||||
const U32 mls = dctx->fParams.mml;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
|
||||
@ -885,9 +885,9 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_decompress_continueDCtx
|
||||
* dctx must have been properly initialized */
|
||||
static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx,
|
||||
/*! ZSTD_decompress_continueDCtx() :
|
||||
* `dctx` must have been properly initialized */
|
||||
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
@ -965,7 +965,7 @@ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDC
|
||||
{
|
||||
ZSTD_copyDCtx(dctx, refDCtx);
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize);
|
||||
return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
@ -977,7 +977,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength);
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize);
|
||||
return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
@ -986,6 +986,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const
|
||||
return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
|
||||
|
||||
@ -71,11 +71,9 @@
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define BLOCKSIZE (128 KB) /* define, for static allocation */
|
||||
#define BLOCKSIZE (128 KB) /* define, for static allocation */
|
||||
|
||||
static const size_t ZSTD_blockHeaderSize = 3;
|
||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||
#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
|
||||
@ -165,13 +165,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||
You can then reuse ZSTD_CCtx to compress some new frame.
|
||||
*/
|
||||
|
||||
typedef struct { U64 frameSize; U32 windowLog; U32 mml; } ZSTD_frameParams;
|
||||
|
||||
#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
|
||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
@ -182,15 +186,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
|
||||
This operation is independent, and just needs enough input data to properly decode the frame header.
|
||||
Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding.
|
||||
Result : 0 when successful, it means the ZSTD_parameters structure has been filled.
|
||||
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
|
||||
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams().
|
||||
It requires to read the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max.
|
||||
If you don't provide enough length, function will return the minimum size it wants to produce a result.
|
||||
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
|
||||
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
|
||||
errorCode, which can be tested using ZSTD_isError()
|
||||
|
||||
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict()
|
||||
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx()
|
||||
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
|
||||
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
|
||||
|
||||
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
|
||||
Loading…
Reference in New Issue
Block a user