diff --git a/.gitignore b/.gitignore index f0e2c9f1..9a71d4bc 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ projects/VS2015 .directory _codelite/ _zstdbench/ -zlib_wrapper/ +.clang_complete # CMake projects/cmake/ diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 6b243c07..88906149 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -102,6 +102,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size incorrect"; case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 660b2d3f..e8cfcc91 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -54,6 +54,7 @@ typedef enum { ZSTD_error_dstSize_tooSmall, ZSTD_error_srcSize_wrong, ZSTD_error_corruption_detected, + ZSTD_error_checksum_wrong, ZSTD_error_tableLog_tooLarge, ZSTD_error_maxSymbolValue_tooLarge, ZSTD_error_maxSymbolValue_tooSmall, diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 596ca03e..e4c992be 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -87,8 +87,9 @@ typedef struct { } ZSTD_compressionParameters; typedef struct { - U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ - U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ + U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ + U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -197,6 +198,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; U32 dictID; + U32 checksumFlag; } ZSTD_frameParams; #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index e078d7eb..66deb495 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -125,7 +125,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; memset(zbc, 0, sizeof(ZBUFF_CCtx)); - zbc->customAlloc = customMem.customAlloc; + zbc->customAlloc = customMem.customAlloc; zbc->customFree = customMem.customFree; zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; @@ -180,6 +180,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); params.fParams.contentSizeFlag = 0; ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c7ba0305..a9a2dc79 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,9 +51,11 @@ /*-************************************* * Dependencies ***************************************/ -#include /* malloc */ -#include /* memset */ +#include /* malloc */ +#include /* memset */ #include "mem.h" +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH_reset, update, digest */ #include "fse_static.h" #include "huf_static.h" #include "zstd_internal.h" @@ -104,6 +106,7 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + XXH64_state_t xxhState; ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; @@ -266,6 +269,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, } } if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ + XXH64_reset(&zc->xxhState, 0); zc->hashTable3 = (U32*)(zc->workSpace); zc->hashTable = zc->hashTable3 + h3Size; zc->chainTable = zc->hashTable + hSize; @@ -1938,7 +1942,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* catch up */ if (offset) { - U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ @@ -2043,19 +2047,22 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa -static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { - size_t blockSize = zc->blockSize; + size_t blockSize = cctx->blockSize; size_t remaining = srcSize; const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - const U32 maxDist = 1 << zc->params.cParams.windowLog; - ZSTD_stats_t* stats = &zc->seqStore.stats; + const U32 maxDist = 1 << cctx->params.cParams.windowLog; + ZSTD_stats_t* stats = &cctx->seqStore.stats; ZSTD_statsInit(stats); + if (cctx->params.fParams.checksumFlag) + XXH64_update(&cctx->xxhState, src, srcSize); + while (remaining) { size_t cSize; ZSTD_statsResetFreqs(stats); @@ -2063,14 +2070,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; - if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { + if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { /* enforce maxDist */ - U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist; - if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit; - if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit; + U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; + if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; + if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; } - cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) { /* block is not compressible */ @@ -2090,7 +2097,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, op += cSize; } - ZSTD_statsPrint(stats, zc->params.cParams.searchLength); + ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); return op-ostart; } @@ -2104,7 +2111,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ | (fcsId << 6) ); U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3); + BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4)); size_t pos; if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); @@ -2261,40 +2268,45 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { /* note : magic number already checked */ - size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode; - short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; - short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + size_t const dictSizeStart = dictSize; - size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize); - if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); - zc->flagStaticTables = 1; - dict = (const char*)dict + hufHeaderSize; - dictSize -= hufHeaderSize; + { size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize); + if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); + zc->flagStaticTables = 1; + dict = (const char*)dict + hufHeaderSize; + dictSize -= hufHeaderSize; + } - offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); - if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + offcodeHeaderSize; - dictSize -= offcodeHeaderSize; + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dict = (const char*)dict + offcodeHeaderSize; + dictSize -= offcodeHeaderSize; + } - matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); - if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + matchlengthHeaderSize; - dictSize -= matchlengthHeaderSize; + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dict = (const char*)dict + matchlengthHeaderSize; + dictSize -= matchlengthHeaderSize; + } - litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); - if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictSize -= litlengthHeaderSize; + } - return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; + return (dictSizeStart-dictSize); } /** ZSTD_compress_insertDictionary() : @@ -2366,30 +2378,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) /*! ZSTD_compressEnd() : * Write frame epilogue. * @return : nb of bytes written into dst (or an error code) */ -size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity) +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) { BYTE* op = (BYTE*)dst; size_t fhSize = 0; /* not even init ! */ - if (zc->stage==0) return ERROR(stage_wrong); + if (cctx->stage==0) return ERROR(stage_wrong); /* special case : empty frame */ - if (zc->stage==1) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0); + if (cctx->stage==1) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; op += fhSize; - zc->stage = 2; + cctx->stage = 2; } /* frame epilogue */ if (dstCapacity < 3) return ERROR(dstSize_tooSmall); - op[0] = (BYTE)(bt_end << 6); - op[1] = 0; - op[2] = 0; + { U32 const checksum = cctx->params.fParams.checksumFlag ? + (U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) : + 0; + op[0] = (BYTE)((bt_end<<6) + (checksum>>16)); + op[1] = (BYTE)(checksum>>8); + op[2] = (BYTE)checksum; + } - zc->stage = 0; /* return to "created by not init" status */ + cctx->stage = 0; /* return to "created but not init" status */ return 3+fhSize; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index da00799c..6d79e678 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -57,6 +57,8 @@ #include /* memcpy, memmove */ #include /* debug only : printf */ #include "mem.h" /* low level memory routines */ +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ #include "zstd_internal.h" #include "fse_static.h" #include "huf_static.h" @@ -117,6 +119,7 @@ struct ZSTD_DCtx_s size_t expected; size_t headerSize; ZSTD_frameParams fParams; + XXH64_state_t xxhState; ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ @@ -146,12 +149,6 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) return 0; } -ZSTD_DCtx* ZSTD_createDCtx(void) -{ - ZSTD_customMem customMem = { NULL, NULL }; - return ZSTD_createDCtx_advanced(customMem); -} - ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; @@ -178,6 +175,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) return dctx; } +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + ZSTD_customMem const customMem = { NULL, NULL }; + return ZSTD_createDCtx_advanced(customMem); +} + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { @@ -346,8 +349,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t BYTE const checkByte = ip[5]; size_t pos = ZSTD_frameHeaderSize_min; U32 const dictIDSizeCode = checkByte&3; - fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->checksumFlag = checkByte & 0x10; switch(dictIDSizeCode) /* fcsId */ { default: /* impossible */ @@ -376,6 +381,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t sr size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); return result; } @@ -964,6 +970,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, if (cBlockSize == 0) break; /* bt_end */ if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); op += decodedSize; ip += cBlockSize; remainingSize -= cBlockSize; @@ -1035,6 +1042,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) return dctx->stage == ZSTDds_skipFrame; } +/** ZSTD_decompressContinue() : +* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) +* or an error code, which can be tested using ZSTD_isError() */ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ @@ -1076,6 +1086,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; if (bp.blockType == bt_end) { + if (dctx->fParams.checksumFlag) { + U64 const h64 = XXH64_digest(&dctx->xxhState); + U32 const h32 = (U32)(h64>>11) & ((1<<22)-1); + const BYTE* const ip = (const BYTE*)src; + U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16); + if (check32 != h32) return ERROR(checksum_wrong); + } dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; } else { @@ -1107,6 +1124,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->stage = ZSTDds_decodeBlockHeader; dctx->expected = ZSTD_blockHeaderSize; dctx->previousDstEnd = (char*)dst + rSize; + if (ZSTD_isError(rSize)) return rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); return rSize; } case ZSTDds_decodeSkippableHeader: @@ -1134,18 +1153,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi dctx->previousDstEnd = (const char*)dict + dictSize; } -static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart) { - size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize; + size_t dictSize = dictSizeStart; - hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); - if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + hSize; - dictSize -= hSize; + { size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); + if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + hSize; + dictSize -= hSize; + } { short offcodeNCount[MaxOff+1]; U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; - offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } @@ -1155,7 +1175,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } @@ -1165,14 +1185,15 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; - litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictSize -= litlengthHeaderSize; } dctx->flagRepeatTable = 1; - return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; + return dictSizeStart - dictSize; } static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) diff --git a/programs/.clang_complete b/programs/.clang_complete deleted file mode 100644 index 658aa00b..00000000 --- a/programs/.clang_complete +++ /dev/null @@ -1,3 +0,0 @@ --I../lib/common --I../lib/legacy --I./legacy diff --git a/programs/.gitignore b/programs/.gitignore index f7061d3b..0f391d2d 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -40,6 +40,7 @@ grillResults.txt _* tmp* *.zst +result # fuzzer afl diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 7fbf906c..0c338928 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -216,7 +216,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = CNBuffSize / 3; { ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize); - ZSTD_frameParameters const fPar = { 1 , 0 }; + ZSTD_frameParameters const fPar = { 1 , 0 , 0 }; ZSTD_parameters p; p.cParams = cPar; p.fParams = fPar; CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); @@ -276,7 +276,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++); - { ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ }; + { ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ }; ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize); ZSTD_parameters p; p.cParams = cParams; p.fParams = fParams; @@ -639,12 +639,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */ dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); - if (FUZ_rand(&lseed) & 15) { + if (FUZ_rand(&lseed) & 0xF) { size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } else { ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); - ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */ + ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, + !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, + 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ ZSTD_parameters p; size_t errorCode; p.cParams = cPar; p.fParams = fpar; diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 278339dd..19a385b4 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -97,7 +97,7 @@ static U32 FUZ_GetMilliStart(void) static U32 FUZ_GetMilliSpan(U32 nTimeStart) { - U32 nCurrent = FUZ_GetMilliStart(); + U32 const nCurrent = FUZ_GetMilliStart(); U32 nSpan = nCurrent - nTimeStart; if (nTimeStart > nCurrent) nSpan += 0x100000 * 1000; @@ -152,7 +152,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); - size_t result, cSize, readSize, readSkipSize, genSize; + size_t cSize, readSize, readSkipSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); @@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); readSize = CNBufferSize; genSize = compressedBufferSize; - result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); - if (ZBUFF_isError(result)) goto _output_error; + { size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); + if (ZBUFF_isError(r)) goto _output_error; } if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ cSize += genSize; genSize = compressedBufferSize - cSize; - result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); - if (result != 0) goto _output_error; /* error, or some data not flushed */ + { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); + if (r != 0) goto _output_error; } /*< error, or some data not flushed */ cSize += genSize; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); @@ -188,13 +188,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSkipSize = cSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); - if (genSize != 0) goto _output_error; /* skippable frame */ + { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); + if (r != 0) goto _output_error; } + if (genSize != 0) goto _output_error; /* skippable frame len is 0 */ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSize = cSize - readSkipSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); - if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ + { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); @@ -613,9 +614,10 @@ int main(int argc, const char** argv) if (testNb==0) { result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */ - if (!result) + if (!result) { + DISPLAYLEVEL(4, "Unit tests using customMem :\n") result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */ - } + } } if (!result) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);