new frame format, allowing custom window size
This commit is contained in:
Yann Collet
parent
89703d20fb
commit
673f0d7cdc
@ -189,8 +189,7 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
|
||||
|
||||
#endif
|
||||
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
@ -180,6 +180,11 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
#define ZSTD_TARGETLENGTH_MIN 4
|
||||
#define ZSTD_TARGETLENGTH_MAX 999
|
||||
|
||||
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
||||
|
||||
|
||||
/*--- Types ---*/
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */
|
||||
@ -305,15 +310,11 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||
|
||||
typedef struct {
|
||||
U64 frameContentSize;
|
||||
U32 windowLog;
|
||||
U32 windowSize;
|
||||
U32 dictID;
|
||||
U32 checksumFlag;
|
||||
} ZSTD_frameParams;
|
||||
|
||||
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||
static const size_t ZSTD_frameHeaderSize_min = 6;
|
||||
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
||||
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
|
||||
@ -81,8 +81,8 @@
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 };
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||
|
||||
@ -128,6 +128,7 @@ struct ZSTD_CCtx_s
|
||||
void* workSpace;
|
||||
size_t workSpaceSize;
|
||||
size_t blockSize;
|
||||
U64 frameContentSize;
|
||||
XXH64_state_t xxhState;
|
||||
ZSTD_customMem customMem;
|
||||
|
||||
@ -257,7 +258,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
|
||||
/*! ZSTD_resetCCtx_advanced() :
|
||||
note : 'params' is expected to be validated */
|
||||
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
ZSTD_parameters params, U32 reset)
|
||||
ZSTD_parameters params, U64 frameContentSize, U32 reset)
|
||||
{ /* note : params considered validated here */
|
||||
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
|
||||
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
|
||||
@ -265,7 +266,10 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
const size_t tokenSpace = blockSize + 11*maxNbSeq;
|
||||
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
|
||||
const size_t hSize = ((size_t)1) << params.cParams.hashLog;
|
||||
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
|
||||
const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 :
|
||||
( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX :
|
||||
((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) );
|
||||
const size_t h3Size = 1 << hashLog3;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
|
||||
/* Check if workSpace is large enough, alloc a new one if needed */
|
||||
@ -282,6 +286,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
|
||||
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
|
||||
XXH64_reset(&zc->xxhState, 0);
|
||||
zc->hashLog3 = hashLog3;
|
||||
zc->hashTable3 = (U32*)(zc->workSpace);
|
||||
zc->hashTable = zc->hashTable3 + h3Size;
|
||||
zc->chainTable = zc->hashTable + hSize;
|
||||
@ -298,6 +303,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
zc->lowLimit = 0;
|
||||
zc->params = params;
|
||||
zc->blockSize = blockSize;
|
||||
zc->frameContentSize = frameContentSize;
|
||||
|
||||
if (params.cParams.strategy == ZSTD_btopt) {
|
||||
zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer);
|
||||
@ -333,9 +339,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
||||
{
|
||||
if (srcCCtx->stage!=1) return ERROR(stage_wrong);
|
||||
|
||||
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
|
||||
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0);
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0);
|
||||
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
|
||||
|
||||
/* copy tables */
|
||||
@ -403,8 +408,9 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
||||
/* Frame format description
|
||||
Frame Header - [ Block Header - Block ] - Frame End
|
||||
1) Frame Header
|
||||
- 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
|
||||
- 1 byte - Frame Descriptor
|
||||
- 4 bytes : Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
|
||||
- 1 byte : Frame Header Descriptor
|
||||
- 1-13 bytes : Optional fields
|
||||
2) Block Header
|
||||
- 3 bytes, starting with a 2-bits descriptor
|
||||
Uncompressed, Compressed, Frame End, unused
|
||||
@ -417,13 +423,38 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
||||
|
||||
/* Frame descriptor
|
||||
|
||||
1 byte, using :
|
||||
// old
|
||||
1 byte - Alloc :
|
||||
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
||||
bit 4 : minmatch 4(0) or 3(1)
|
||||
bit 4 : reserved for windowLog (must be zero)
|
||||
bit 5 : reserved (must be zero)
|
||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||
|
||||
Optional : content size (0, 1, 2 or 8 bytes)
|
||||
1 byte - checker :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-7 : reserved (must be zero)
|
||||
|
||||
|
||||
// new
|
||||
1 byte - FrameHeaderDescription :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-4 : reserved (must be zero)
|
||||
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
|
||||
bit 6-7 : FrameContentFieldsize (0, 2, 4, or 8)
|
||||
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||
|
||||
Optional : WindowLog (0 or 1 byte)
|
||||
bit 0-2 : octal Fractional (1/8th)
|
||||
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||
|
||||
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||
Automatic adaptation
|
||||
0 : no dictID
|
||||
1 : 1 - 255
|
||||
2 : 256 - 65535
|
||||
4 : all other values
|
||||
|
||||
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||
0 : unknown
|
||||
1 : 0-255 bytes
|
||||
2 : 256 - 65535+256
|
||||
@ -2116,21 +2147,22 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
|
||||
{ BYTE* const op = (BYTE*)dst;
|
||||
U32 const fcsId = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */
|
||||
0;
|
||||
BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
|
||||
| (fcsId << 6) );
|
||||
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4));
|
||||
U32 const checksumFlag = params.fParams.checksumFlag;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */
|
||||
0;
|
||||
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (directModeFlag<<5) + (fcsCode<<6) );
|
||||
size_t pos;
|
||||
|
||||
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
||||
|
||||
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
op[4] = fAllocByte;
|
||||
op[5] = fCheckByte;
|
||||
pos = 6;
|
||||
op[4] = frameHeaderDecriptionByte; pos=5;
|
||||
if (!directModeFlag) op[pos++] = windowLogByte;
|
||||
switch(dictIDSizeCode)
|
||||
{
|
||||
default: /* impossible */
|
||||
@ -2139,12 +2171,12 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break;
|
||||
case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
|
||||
}
|
||||
switch(fcsId)
|
||||
switch(fcsCode)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : break;
|
||||
case 1 : op[pos] = (BYTE)(pledgedSrcSize); pos++; break;
|
||||
case 2 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
|
||||
case 0 : if (directModeFlag) op[pos++] = (BYTE)(pledgedSrcSize); break;
|
||||
case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
|
||||
case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
|
||||
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
|
||||
}
|
||||
return pos;
|
||||
@ -2161,7 +2193,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
|
||||
|
||||
if (zc->stage==0) return ERROR(stage_wrong);
|
||||
if (frame && (zc->stage==1)) { /* copy saved header */
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize, zc->dictID);
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
|
||||
if (ZSTD_isError(fhSize)) return fhSize;
|
||||
dstCapacity -= fhSize;
|
||||
dst = (char*)dst + fhSize;
|
||||
@ -2344,11 +2376,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
{ U32 const hashLog3 = (!pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
|
||||
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; }
|
||||
|
||||
{ size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1);
|
||||
if (ZSTD_isError(resetError)) return resetError; }
|
||||
size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1);
|
||||
if (ZSTD_isError(resetError)) return resetError;
|
||||
|
||||
return ZSTD_compress_insertDictionary(zc, dict, dictSize);
|
||||
}
|
||||
|
||||
@ -191,10 +191,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
if (ZSTD_isError(h2Result)) return h2Result;
|
||||
} }
|
||||
|
||||
if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */
|
||||
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
|
||||
{ size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||
zbd->blockSize = blockSize;
|
||||
if (zbd->inBuffSize < blockSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
@ -202,7 +202,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
|
||||
{ size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
|
||||
@ -64,7 +64,7 @@
|
||||
#include "huf.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||
# include "zstd_legacy.h"
|
||||
#endif
|
||||
|
||||
@ -218,22 +218,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-7 : reserved (must be zero)
|
||||
|
||||
|
||||
// new
|
||||
1 byte - Alloc :
|
||||
1 byte - FrameHeaderDescription :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-4 : reserved (must be zero)
|
||||
bit 5 : WindowLog skipped (note : if 1, then fcs > 0)
|
||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
|
||||
bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8)
|
||||
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||
|
||||
OR : 0-3 : no windowLog, 1, 2, 4, 8
|
||||
4-7 : windowLog, 0, 2, 4, 8
|
||||
|
||||
1 byte - WindowLog (can be skipped if fcs>0)
|
||||
Optional : WindowLog (0 or 1 byte)
|
||||
bit 0-2 : octal Fractional (1/8th)
|
||||
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||
|
||||
|
||||
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||
Automatic adaptation
|
||||
0 : no dictID
|
||||
@ -241,11 +237,12 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
2 : 256 - 65535
|
||||
4 : all other values
|
||||
|
||||
Optional : content size (0, 1, 2 or 8 bytes)
|
||||
0 : unknown
|
||||
1 : 0-255 bytes
|
||||
2 : 256 - 65535+256
|
||||
8 : up to 16 exa
|
||||
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||
0 : unknown (fcfs==0 and swl==0)
|
||||
1 : 0-255 bytes (fcfs==0 and swl==1)
|
||||
2 : 256 - 65535+256 (fcfs==1)
|
||||
4 : 0 - 4GB-1 (fcfs==2)
|
||||
8 : 0 - 16EB-1 (fcfs==3)
|
||||
*/
|
||||
|
||||
|
||||
@ -321,15 +318,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
{ U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
|
||||
U32 const dictID =(((const BYTE*)src)[5]) & 3;
|
||||
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId] + ZSTD_did_fieldSize[dictID];
|
||||
{ BYTE const fhd = ((const BYTE*)src)[4];
|
||||
U32 const dictID= fhd & 3;
|
||||
U32 const directMode = (fhd >> 5) & 1;
|
||||
U32 const fcsId = fhd >> 6;
|
||||
return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
|
||||
+ (directMode && !ZSTD_fcs_fieldSize[fcsId]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_getFrameParams() :
|
||||
* decode Frame Header, or provide expected `srcSize`.
|
||||
* decode Frame Header, or require larger `srcSize`.
|
||||
* @return : 0, `fparamsPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, result is expected `srcSize`,
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
@ -343,7 +343,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||
fparamsPtr->windowLog = 0; /* windowLog==0 means a frame is skippable */
|
||||
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||
return 0;
|
||||
}
|
||||
return ERROR(prefix_unknown);
|
||||
@ -353,31 +353,48 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
|
||||
if (srcSize < fhsize) return fhsize; }
|
||||
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
{ BYTE const allocByte = ip[4];
|
||||
BYTE const checkByte = ip[5];
|
||||
size_t pos = ZSTD_frameHeaderSize_min;
|
||||
U32 const dictIDSizeCode = checkByte&3;
|
||||
if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
|
||||
if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
|
||||
fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
fparamsPtr->checksumFlag = checkByte & 0x10;
|
||||
switch(dictIDSizeCode) /* fcsId */
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : fparamsPtr->dictID = 0; break;
|
||||
case 1 : fparamsPtr->dictID = ip[pos]; pos++; break;
|
||||
case 2 : fparamsPtr->dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
||||
case 3 : fparamsPtr->dictID = MEM_readLE32(ip+pos); pos+=4; break;
|
||||
{ BYTE const fhdByte = ip[4];
|
||||
size_t pos = 5;
|
||||
U32 const dictIDSizeCode = fhdByte&3;
|
||||
U32 const checksumFlag = (fhdByte>>2)&1;
|
||||
U32 const directMode = (fhdByte>>5)&1;
|
||||
U32 const fcsID = fhdByte>>6;
|
||||
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
|
||||
U32 windowSize = 0;
|
||||
U32 dictID = 0;
|
||||
U64 frameContentSize = 0;
|
||||
if ((fhdByte & 0x18) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
|
||||
if (!directMode) {
|
||||
BYTE const wlByte = ip[pos++];
|
||||
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
||||
windowSize = (1U << windowLog);
|
||||
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||
}
|
||||
switch(allocByte >> 6) /* fcsId */
|
||||
|
||||
switch(dictIDSizeCode)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : fparamsPtr->frameContentSize = 0; break;
|
||||
case 1 : fparamsPtr->frameContentSize = ip[pos]; break;
|
||||
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
||||
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+pos); break;
|
||||
} }
|
||||
case 0 : break;
|
||||
case 1 : dictID = ip[pos]; pos++; break;
|
||||
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
||||
case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
|
||||
}
|
||||
switch(fcsID)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : if (directMode) frameContentSize = ip[pos]; break;
|
||||
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
||||
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
|
||||
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||
}
|
||||
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
||||
fparamsPtr->frameContentSize = frameContentSize;
|
||||
fparamsPtr->windowSize = windowSize;
|
||||
fparamsPtr->dictID = dictID;
|
||||
fparamsPtr->checksumFlag = checksumFlag;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -388,7 +405,6 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
|
||||
if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
||||
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
||||
return result;
|
||||
@ -1059,7 +1075,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
||||
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
||||
|
||||
/* Decompress : frame header; part 1 */
|
||||
switch (dctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
|
||||
@ -615,7 +615,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
||||
|
||||
/* Header loading (optional, saves one loop) */
|
||||
{ size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
|
||||
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
|
||||
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
|
||||
readSize = alreadyLoaded + loadedSize;
|
||||
}
|
||||
|
||||
@ -23,7 +23,7 @@ roundTripTest() {
|
||||
isWindows=false
|
||||
ECHO="echo"
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
Windows*)
|
||||
isWindows=true
|
||||
ECHO="echo -e"
|
||||
;;
|
||||
@ -42,8 +42,8 @@ file $ZSTD
|
||||
$ECHO "\n**** simple tests **** "
|
||||
|
||||
./datagen > tmp
|
||||
$ZSTD -f tmp # trivial compression case, creates tmp.zst
|
||||
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
|
||||
$ZSTD -f tmp # trivial compression case, creates tmp.zst
|
||||
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
|
||||
$ECHO "test : too large compression level (must fail)"
|
||||
$ZSTD -99 tmp && die "too large compression level undetected"
|
||||
$ECHO "test : compress to stdout"
|
||||
@ -57,7 +57,7 @@ $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
|
||||
$ZSTD --decompress tmpCompressed -c > tmpResult
|
||||
$ZSTD --decompress tmpCompressed --stdout > tmpResult
|
||||
if [ "$isWindows" = false ] ; then
|
||||
$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout
|
||||
$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout
|
||||
$ZSTD -d - < tmp.zst > /dev/null
|
||||
fi
|
||||
$ZSTD -dc < tmp.zst > /dev/null
|
||||
|
||||
@ -171,6 +171,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||
cSize = skippableFrameSize + 8;
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
|
||||
@ -186,14 +187,18 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
cSize += genSize;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* Basic decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
/* skippable frame test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSkipSize = cSize;
|
||||
genSize = CNBufferSize;
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
|
||||
if (r != 0) goto _output_error; }
|
||||
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Basic decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSize = cSize - readSkipSize;
|
||||
genSize = CNBufferSize;
|
||||
@ -204,13 +209,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Byte-by-byte decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
|
||||
@ -234,13 +238,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
_end:
|
||||
ZBUFF_freeCCtx(zc);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user