Merge remote-tracking branch 'refs/remotes/Cyan4973/dev070' into dev070
# Conflicts: # .gitignore # lib/decompress/zstd_decompress.c # programs/zbufftest.c
This commit is contained in:
inikep
commit
5c2771710d
2
.gitignore
vendored
2
.gitignore
vendored
@ -31,7 +31,7 @@ projects/VS2015
|
||||
.directory
|
||||
_codelite/
|
||||
_zstdbench/
|
||||
zlib_wrapper/
|
||||
.clang_complete
|
||||
|
||||
# CMake
|
||||
projects/cmake/
|
||||
|
||||
@ -102,6 +102,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size incorrect";
|
||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
|
||||
@ -54,6 +54,7 @@ typedef enum {
|
||||
ZSTD_error_dstSize_tooSmall,
|
||||
ZSTD_error_srcSize_wrong,
|
||||
ZSTD_error_corruption_detected,
|
||||
ZSTD_error_checksum_wrong,
|
||||
ZSTD_error_tableLog_tooLarge,
|
||||
ZSTD_error_maxSymbolValue_tooLarge,
|
||||
ZSTD_error_maxSymbolValue_tooSmall,
|
||||
|
||||
@ -87,8 +87,9 @@ typedef struct {
|
||||
} ZSTD_compressionParameters;
|
||||
|
||||
typedef struct {
|
||||
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
|
||||
U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
|
||||
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
|
||||
U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
|
||||
U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
|
||||
} ZSTD_frameParameters;
|
||||
|
||||
typedef struct {
|
||||
@ -197,6 +198,7 @@ typedef struct {
|
||||
U64 frameContentSize;
|
||||
U32 windowLog;
|
||||
U32 dictID;
|
||||
U32 checksumFlag;
|
||||
} ZSTD_frameParams;
|
||||
|
||||
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||
|
||||
@ -125,7 +125,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
|
||||
zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx));
|
||||
if (zbc==NULL) return NULL;
|
||||
memset(zbc, 0, sizeof(ZBUFF_CCtx));
|
||||
zbc->customAlloc = customMem.customAlloc;
|
||||
zbc->customAlloc = customMem.customAlloc;
|
||||
zbc->customFree = customMem.customFree;
|
||||
zbc->zc = ZSTD_createCCtx_advanced(customMem);
|
||||
return zbc;
|
||||
@ -180,6 +180,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
|
||||
@ -51,9 +51,11 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <string.h> /* memset */
|
||||
#include "mem.h"
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH_reset, update, digest */
|
||||
#include "fse_static.h"
|
||||
#include "huf_static.h"
|
||||
#include "zstd_internal.h"
|
||||
@ -104,6 +106,7 @@ struct ZSTD_CCtx_s
|
||||
void* workSpace;
|
||||
size_t workSpaceSize;
|
||||
size_t blockSize;
|
||||
XXH64_state_t xxhState;
|
||||
ZSTD_allocFunction customAlloc;
|
||||
ZSTD_freeFunction customFree;
|
||||
|
||||
@ -266,6 +269,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
} }
|
||||
|
||||
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
|
||||
XXH64_reset(&zc->xxhState, 0);
|
||||
zc->hashTable3 = (U32*)(zc->workSpace);
|
||||
zc->hashTable = zc->hashTable3 + h3Size;
|
||||
zc->chainTable = zc->hashTable + hSize;
|
||||
@ -1938,7 +1942,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
|
||||
/* catch up */
|
||||
if (offset) {
|
||||
U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
|
||||
U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
|
||||
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
|
||||
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
|
||||
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
||||
@ -2043,19 +2047,22 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
|
||||
|
||||
|
||||
|
||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
size_t blockSize = zc->blockSize;
|
||||
size_t blockSize = cctx->blockSize;
|
||||
size_t remaining = srcSize;
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
const U32 maxDist = 1 << zc->params.cParams.windowLog;
|
||||
ZSTD_stats_t* stats = &zc->seqStore.stats;
|
||||
const U32 maxDist = 1 << cctx->params.cParams.windowLog;
|
||||
ZSTD_stats_t* stats = &cctx->seqStore.stats;
|
||||
ZSTD_statsInit(stats);
|
||||
|
||||
if (cctx->params.fParams.checksumFlag)
|
||||
XXH64_update(&cctx->xxhState, src, srcSize);
|
||||
|
||||
while (remaining) {
|
||||
size_t cSize;
|
||||
ZSTD_statsResetFreqs(stats);
|
||||
@ -2063,14 +2070,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
||||
if (remaining < blockSize) blockSize = remaining;
|
||||
|
||||
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
|
||||
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
|
||||
/* enforce maxDist */
|
||||
U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
|
||||
if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit;
|
||||
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
|
||||
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
|
||||
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
|
||||
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
|
||||
}
|
||||
|
||||
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
|
||||
cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
|
||||
if (cSize == 0) { /* block is not compressible */
|
||||
@ -2090,7 +2097,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
|
||||
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
@ -2104,7 +2111,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
|
||||
| (fcsId << 6) );
|
||||
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3);
|
||||
BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4));
|
||||
size_t pos;
|
||||
|
||||
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
||||
@ -2261,40 +2268,45 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
|
||||
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
||||
{
|
||||
/* note : magic number already checked */
|
||||
size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode;
|
||||
short offcodeNCount[MaxOff+1];
|
||||
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
|
||||
short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
size_t const dictSizeStart = dictSize;
|
||||
|
||||
size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
|
||||
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
zc->flagStaticTables = 1;
|
||||
dict = (const char*)dict + hufHeaderSize;
|
||||
dictSize -= hufHeaderSize;
|
||||
{ size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
|
||||
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
zc->flagStaticTables = 1;
|
||||
dict = (const char*)dict + hufHeaderSize;
|
||||
dictSize -= hufHeaderSize;
|
||||
}
|
||||
|
||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + offcodeHeaderSize;
|
||||
dictSize -= offcodeHeaderSize;
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
|
||||
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dict = (const char*)dict + offcodeHeaderSize;
|
||||
dictSize -= offcodeHeaderSize;
|
||||
}
|
||||
|
||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + matchlengthHeaderSize;
|
||||
dictSize -= matchlengthHeaderSize;
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dict = (const char*)dict + matchlengthHeaderSize;
|
||||
dictSize -= matchlengthHeaderSize;
|
||||
}
|
||||
|
||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictSize -= litlengthHeaderSize;
|
||||
}
|
||||
|
||||
return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
||||
return (dictSizeStart-dictSize);
|
||||
}
|
||||
|
||||
/** ZSTD_compress_insertDictionary() :
|
||||
@ -2366,30 +2378,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
|
||||
/*! ZSTD_compressEnd() :
|
||||
* Write frame epilogue.
|
||||
* @return : nb of bytes written into dst (or an error code) */
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity)
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t fhSize = 0;
|
||||
|
||||
/* not even init ! */
|
||||
if (zc->stage==0) return ERROR(stage_wrong);
|
||||
if (cctx->stage==0) return ERROR(stage_wrong);
|
||||
|
||||
/* special case : empty frame */
|
||||
if (zc->stage==1) {
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0);
|
||||
if (cctx->stage==1) {
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
|
||||
if (ZSTD_isError(fhSize)) return fhSize;
|
||||
dstCapacity -= fhSize;
|
||||
op += fhSize;
|
||||
zc->stage = 2;
|
||||
cctx->stage = 2;
|
||||
}
|
||||
|
||||
/* frame epilogue */
|
||||
if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
|
||||
op[0] = (BYTE)(bt_end << 6);
|
||||
op[1] = 0;
|
||||
op[2] = 0;
|
||||
{ U32 const checksum = cctx->params.fParams.checksumFlag ?
|
||||
(U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) :
|
||||
0;
|
||||
op[0] = (BYTE)((bt_end<<6) + (checksum>>16));
|
||||
op[1] = (BYTE)(checksum>>8);
|
||||
op[2] = (BYTE)checksum;
|
||||
}
|
||||
|
||||
zc->stage = 0; /* return to "created by not init" status */
|
||||
cctx->stage = 0; /* return to "created but not init" status */
|
||||
return 3+fhSize;
|
||||
}
|
||||
|
||||
|
||||
@ -57,6 +57,8 @@
|
||||
#include <string.h> /* memcpy, memmove */
|
||||
#include <stdio.h> /* debug only : printf */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#include "zstd_internal.h"
|
||||
#include "fse_static.h"
|
||||
#include "huf_static.h"
|
||||
@ -117,6 +119,7 @@ struct ZSTD_DCtx_s
|
||||
size_t expected;
|
||||
size_t headerSize;
|
||||
ZSTD_frameParams fParams;
|
||||
XXH64_state_t xxhState;
|
||||
ZSTD_allocFunction customAlloc;
|
||||
ZSTD_freeFunction customFree;
|
||||
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||
@ -146,12 +149,6 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTD_createDCtx(void)
|
||||
{
|
||||
ZSTD_customMem customMem = { NULL, NULL };
|
||||
return ZSTD_createDCtx_advanced(customMem);
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
ZSTD_DCtx* dctx;
|
||||
@ -178,6 +175,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
|
||||
return dctx;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTD_createDCtx(void)
|
||||
{
|
||||
ZSTD_customMem const customMem = { NULL, NULL };
|
||||
return ZSTD_createDCtx_advanced(customMem);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
||||
{
|
||||
@ -346,8 +349,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
BYTE const checkByte = ip[5];
|
||||
size_t pos = ZSTD_frameHeaderSize_min;
|
||||
U32 const dictIDSizeCode = checkByte&3;
|
||||
fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
|
||||
if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
|
||||
fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
fparamsPtr->checksumFlag = checkByte & 0x10;
|
||||
switch(dictIDSizeCode) /* fcsId */
|
||||
{
|
||||
default: /* impossible */
|
||||
@ -376,6 +381,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t sr
|
||||
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
|
||||
if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
||||
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -964,6 +970,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||
op += decodedSize;
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
@ -1035,6 +1042,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
|
||||
return dctx->stage == ZSTDds_skipFrame;
|
||||
}
|
||||
|
||||
/** ZSTD_decompressContinue() :
|
||||
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
@ -1076,6 +1086,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
if (bp.blockType == bt_end) {
|
||||
if (dctx->fParams.checksumFlag) {
|
||||
U64 const h64 = XXH64_digest(&dctx->xxhState);
|
||||
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
|
||||
const BYTE* const ip = (const BYTE*)src;
|
||||
U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
|
||||
if (check32 != h32) return ERROR(checksum_wrong);
|
||||
}
|
||||
dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
} else {
|
||||
@ -1107,6 +1124,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
dctx->expected = ZSTD_blockHeaderSize;
|
||||
dctx->previousDstEnd = (char*)dst + rSize;
|
||||
if (ZSTD_isError(rSize)) return rSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
|
||||
return rSize;
|
||||
}
|
||||
case ZSTDds_decodeSkippableHeader:
|
||||
@ -1134,18 +1153,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
|
||||
dctx->previousDstEnd = (const char*)dict + dictSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart)
|
||||
{
|
||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
|
||||
size_t dictSize = dictSizeStart;
|
||||
|
||||
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
||||
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + hSize;
|
||||
dictSize -= hSize;
|
||||
{ size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
||||
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + hSize;
|
||||
dictSize -= hSize;
|
||||
}
|
||||
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
@ -1155,7 +1175,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
|
||||
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
@ -1165,14 +1185,15 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
|
||||
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictSize -= litlengthHeaderSize;
|
||||
}
|
||||
|
||||
dctx->flagRepeatTable = 1;
|
||||
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
||||
return dictSizeStart - dictSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
-I../lib/common
|
||||
-I../lib/legacy
|
||||
-I./legacy
|
||||
1
programs/.gitignore
vendored
1
programs/.gitignore
vendored
@ -40,6 +40,7 @@ grillResults.txt
|
||||
_*
|
||||
tmp*
|
||||
*.zst
|
||||
result
|
||||
|
||||
# fuzzer
|
||||
afl
|
||||
|
||||
@ -216,7 +216,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
|
||||
{ size_t const testSize = CNBuffSize / 3;
|
||||
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
|
||||
ZSTD_frameParameters const fPar = { 1 , 0 };
|
||||
ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
|
||||
ZSTD_parameters p;
|
||||
p.cParams = cPar; p.fParams = fPar;
|
||||
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
|
||||
@ -276,7 +276,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
|
||||
{ ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ };
|
||||
{ ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
|
||||
ZSTD_parameters p;
|
||||
p.cParams = cParams; p.fParams = fParams;
|
||||
@ -639,12 +639,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
|
||||
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
|
||||
|
||||
if (FUZ_rand(&lseed) & 15) {
|
||||
if (FUZ_rand(&lseed) & 0xF) {
|
||||
size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
|
||||
} else {
|
||||
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */
|
||||
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
|
||||
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
|
||||
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
|
||||
ZSTD_parameters p;
|
||||
size_t errorCode;
|
||||
p.cParams = cPar; p.fParams = fpar;
|
||||
|
||||
@ -97,7 +97,7 @@ static U32 FUZ_GetMilliStart(void)
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 nCurrent = FUZ_GetMilliStart();
|
||||
U32 const nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
@ -152,7 +152,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
void* compressedBuffer = malloc(compressedBufferSize);
|
||||
size_t const decodedBufferSize = CNBufferSize;
|
||||
void* decodedBuffer = malloc(decodedBufferSize);
|
||||
size_t result, cSize, readSize, readSkipSize, genSize;
|
||||
size_t cSize, readSize, readSkipSize, genSize;
|
||||
U32 testNb=0;
|
||||
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
|
||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
|
||||
@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
|
||||
readSize = CNBufferSize;
|
||||
genSize = compressedBufferSize;
|
||||
result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
|
||||
if (ZBUFF_isError(result)) goto _output_error;
|
||||
{ size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
|
||||
if (ZBUFF_isError(r)) goto _output_error; }
|
||||
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
|
||||
cSize += genSize;
|
||||
genSize = compressedBufferSize - cSize;
|
||||
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||
if (result != 0) goto _output_error; /* error, or some data not flushed */
|
||||
{ size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
|
||||
cSize += genSize;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
@ -188,13 +188,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSkipSize = cSize;
|
||||
genSize = CNBufferSize;
|
||||
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
|
||||
if (genSize != 0) goto _output_error; /* skippable frame */
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
|
||||
if (r != 0) goto _output_error; }
|
||||
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSize = cSize - readSkipSize;
|
||||
genSize = CNBufferSize;
|
||||
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
|
||||
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
|
||||
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
@ -613,9 +614,10 @@ int main(int argc, const char** argv)
|
||||
|
||||
if (testNb==0) {
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
||||
if (!result)
|
||||
if (!result) {
|
||||
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
|
||||
}
|
||||
} }
|
||||
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user