Merge branch 'dev' into oss
This commit is contained in:
commit
826b555463
3
.gitignore
vendored
3
.gitignore
vendored
@ -31,6 +31,7 @@ projects/
|
|||||||
bin/
|
bin/
|
||||||
.buckd/
|
.buckd/
|
||||||
buck-out/
|
buck-out/
|
||||||
|
build-*
|
||||||
|
|
||||||
# Other files
|
# Other files
|
||||||
.directory
|
.directory
|
||||||
@ -43,3 +44,5 @@ _zstdbench/
|
|||||||
googletest/
|
googletest/
|
||||||
*.d
|
*.d
|
||||||
*.vscode
|
*.vscode
|
||||||
|
compile_commands.json
|
||||||
|
.clangd
|
@ -49,6 +49,9 @@ target_link_libraries(fullbench libzstd_static)
|
|||||||
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
|
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
|
||||||
target_link_libraries(fuzzer libzstd_static)
|
target_link_libraries(fuzzer libzstd_static)
|
||||||
|
|
||||||
|
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
|
||||||
|
target_link_libraries(zstreamtest libzstd_static)
|
||||||
|
|
||||||
if (UNIX)
|
if (UNIX)
|
||||||
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
|
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
|
||||||
target_link_libraries(paramgrill libzstd_static m) #m is math library
|
target_link_libraries(paramgrill libzstd_static m) #m is math library
|
||||||
|
@ -161,7 +161,7 @@ ifneq (,$(filter Windows%,$(OS)))
|
|||||||
LIBZSTD = dll\libzstd.dll
|
LIBZSTD = dll\libzstd.dll
|
||||||
$(LIBZSTD): $(ZSTD_FILES)
|
$(LIBZSTD): $(ZSTD_FILES)
|
||||||
@echo compiling dynamic library $(LIBVER)
|
@echo compiling dynamic library $(LIBVER)
|
||||||
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
|
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@
|
||||||
|
|
||||||
else
|
else
|
||||||
|
|
||||||
|
@ -1336,10 +1336,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
|
|||||||
|
|
||||||
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
|
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
|
||||||
if (forceResetIndex == ZSTDirp_reset) {
|
if (forceResetIndex == ZSTDirp_reset) {
|
||||||
memset(&ms->window, 0, sizeof(ms->window));
|
ZSTD_window_init(&ms->window);
|
||||||
ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */
|
|
||||||
ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
|
|
||||||
ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */
|
|
||||||
ZSTD_cwksp_mark_tables_dirty(ws);
|
ZSTD_cwksp_mark_tables_dirty(ws);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1432,7 +1429,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
|||||||
size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1);
|
size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1);
|
||||||
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
|
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
|
||||||
|
|
||||||
ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue;
|
ZSTD_indexResetPolicy_e needsIndexReset = zc->initialized ? ZSTDirp_continue : ZSTDirp_reset;
|
||||||
|
|
||||||
if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) {
|
if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) {
|
||||||
needsIndexReset = ZSTDirp_reset;
|
needsIndexReset = ZSTDirp_reset;
|
||||||
@ -1557,11 +1554,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
|||||||
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
|
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
|
||||||
zc->maxNbLdmSequences = maxNbLdmSeq;
|
zc->maxNbLdmSequences = maxNbLdmSeq;
|
||||||
|
|
||||||
memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window));
|
ZSTD_window_init(&zc->ldmState.window);
|
||||||
ZSTD_window_clear(&zc->ldmState.window);
|
ZSTD_window_clear(&zc->ldmState.window);
|
||||||
}
|
}
|
||||||
|
|
||||||
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
|
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
|
||||||
|
zc->initialized = 1;
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@ -2593,23 +2591,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
|
|||||||
/* Ensure hash/chain table insertion resumes no sooner than lowlimit */
|
/* Ensure hash/chain table insertion resumes no sooner than lowlimit */
|
||||||
if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
|
if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
|
||||||
|
|
||||||
{ int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams);
|
{ size_t cSize;
|
||||||
size_t cSize = 0;
|
int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams);
|
||||||
if (useTargetCBlockSize) {
|
if (useTargetCBlockSize) {
|
||||||
cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
|
cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
|
||||||
FORWARD_IF_ERROR(cSize);
|
FORWARD_IF_ERROR(cSize);
|
||||||
} else {
|
} else {
|
||||||
cSize = ZSTD_compressBlock_internal(cctx,
|
cSize = ZSTD_compressBlock_internal(cctx,
|
||||||
op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
|
op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
|
||||||
ip, blockSize, 0 /* frame */);
|
ip, blockSize, 1 /* frame */);
|
||||||
FORWARD_IF_ERROR(cSize);
|
FORWARD_IF_ERROR(cSize);
|
||||||
|
|
||||||
if (cSize == 0) { /* block is not compressible */
|
if (cSize == 0) { /* block is not compressible */
|
||||||
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
|
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
|
||||||
FORWARD_IF_ERROR(cSize);
|
FORWARD_IF_ERROR(cSize);
|
||||||
} else {
|
} else {
|
||||||
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
|
U32 const cBlockHeader = cSize == 1 ?
|
||||||
MEM_writeLE24(op, cBlockHeader24);
|
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
|
||||||
|
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
|
||||||
|
MEM_writeLE24(op, cBlockHeader);
|
||||||
cSize += ZSTD_blockHeaderSize;
|
cSize += ZSTD_blockHeaderSize;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -3767,11 +3767,11 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
ZSTD_EndDirective const flushMode)
|
ZSTD_EndDirective const flushMode)
|
||||||
{
|
{
|
||||||
const char* const istart = (const char*)input->src;
|
const char* const istart = (const char*)input->src;
|
||||||
const char* const iend = istart + input->size;
|
const char* const iend = input->size != 0 ? istart + input->size : istart;
|
||||||
const char* ip = istart + input->pos;
|
const char* ip = input->pos != 0 ? istart + input->pos : istart;
|
||||||
char* const ostart = (char*)output->dst;
|
char* const ostart = (char*)output->dst;
|
||||||
char* const oend = ostart + output->size;
|
char* const oend = output->size != 0 ? ostart + output->size : ostart;
|
||||||
char* op = ostart + output->pos;
|
char* op = output->pos != 0 ? ostart + output->pos : ostart;
|
||||||
U32 someMoreWork = 1;
|
U32 someMoreWork = 1;
|
||||||
|
|
||||||
/* check expectations */
|
/* check expectations */
|
||||||
@ -3810,7 +3810,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
zcs->inBuff + zcs->inBuffPos, toLoad,
|
zcs->inBuff + zcs->inBuffPos, toLoad,
|
||||||
ip, iend-ip);
|
ip, iend-ip);
|
||||||
zcs->inBuffPos += loaded;
|
zcs->inBuffPos += loaded;
|
||||||
ip += loaded;
|
if (loaded != 0)
|
||||||
|
ip += loaded;
|
||||||
if ( (flushMode == ZSTD_e_continue)
|
if ( (flushMode == ZSTD_e_continue)
|
||||||
&& (zcs->inBuffPos < zcs->inBuffTarget) ) {
|
&& (zcs->inBuffPos < zcs->inBuffTarget) ) {
|
||||||
/* not enough input to fill full block : stop here */
|
/* not enough input to fill full block : stop here */
|
||||||
@ -3870,7 +3871,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
|
zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
|
||||||
DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
|
DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
|
||||||
(unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
|
(unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
|
||||||
op += flushed;
|
if (flushed)
|
||||||
|
op += flushed;
|
||||||
zcs->outBuffFlushedSize += flushed;
|
zcs->outBuffFlushedSize += flushed;
|
||||||
if (toFlush!=flushed) {
|
if (toFlush!=flushed) {
|
||||||
/* flush not fully completed, presumably because dst is too small */
|
/* flush not fully completed, presumably because dst is too small */
|
||||||
|
@ -249,6 +249,7 @@ struct ZSTD_CCtx_s {
|
|||||||
size_t staticSize;
|
size_t staticSize;
|
||||||
SeqCollector seqCollector;
|
SeqCollector seqCollector;
|
||||||
int isFirstBlock;
|
int isFirstBlock;
|
||||||
|
int initialized;
|
||||||
|
|
||||||
seqStore_t seqStore; /* sequences storage ptrs */
|
seqStore_t seqStore; /* sequences storage ptrs */
|
||||||
ldmState_t ldmState; /* long distance matching state */
|
ldmState_t ldmState; /* long distance matching state */
|
||||||
@ -858,6 +859,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
|
|||||||
} } }
|
} } }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
|
||||||
|
memset(window, 0, sizeof(*window));
|
||||||
|
window->base = (BYTE const*)"";
|
||||||
|
window->dictBase = (BYTE const*)"";
|
||||||
|
window->dictLimit = 1; /* start from 1, so that 1st position is valid */
|
||||||
|
window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
|
||||||
|
window->nextSrc = window->base + 1; /* see issue #1241 */
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ZSTD_window_update():
|
* ZSTD_window_update():
|
||||||
* Updates the window by appending [src, src + srcSize) to the window.
|
* Updates the window by appending [src, src + srcSize) to the window.
|
||||||
@ -871,6 +881,10 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
|||||||
BYTE const* const ip = (BYTE const*)src;
|
BYTE const* const ip = (BYTE const*)src;
|
||||||
U32 contiguous = 1;
|
U32 contiguous = 1;
|
||||||
DEBUGLOG(5, "ZSTD_window_update");
|
DEBUGLOG(5, "ZSTD_window_update");
|
||||||
|
if (srcSize == 0)
|
||||||
|
return contiguous;
|
||||||
|
assert(window->base != NULL);
|
||||||
|
assert(window->dictBase != NULL);
|
||||||
/* Check if blocks follow each other */
|
/* Check if blocks follow each other */
|
||||||
if (src != window->nextSrc) {
|
if (src != window->nextSrc) {
|
||||||
/* not contiguous */
|
/* not contiguous */
|
||||||
|
@ -96,7 +96,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
|
|||||||
dictCParams->hashLog : hBitsL;
|
dictCParams->hashLog : hBitsL;
|
||||||
const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ?
|
const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ?
|
||||||
dictCParams->chainLog : hBitsS;
|
dictCParams->chainLog : hBitsS;
|
||||||
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart);
|
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
|
||||||
|
|
||||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic");
|
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic");
|
||||||
|
|
||||||
@ -271,7 +271,7 @@ _match_stored:
|
|||||||
U32 const repIndex2 = current2 - offset_2;
|
U32 const repIndex2 = current2 - offset_2;
|
||||||
const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState
|
const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState
|
||||||
&& repIndex2 < prefixLowestIndex ?
|
&& repIndex2 < prefixLowestIndex ?
|
||||||
dictBase - dictIndexDelta + repIndex2 :
|
dictBase + repIndex2 - dictIndexDelta :
|
||||||
base + repIndex2;
|
base + repIndex2;
|
||||||
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
||||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||||
|
@ -660,7 +660,7 @@ ZSTD_compressBlock_lazy_generic(
|
|||||||
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
|
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
|
||||||
prefixLowestIndex - (U32)(dictEnd - dictBase) :
|
prefixLowestIndex - (U32)(dictEnd - dictBase) :
|
||||||
0;
|
0;
|
||||||
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest);
|
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
ip += (dictAndPrefixLength == 0);
|
ip += (dictAndPrefixLength == 0);
|
||||||
|
@ -490,7 +490,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
|
|||||||
/* Size the seq pool tables */
|
/* Size the seq pool tables */
|
||||||
ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
|
ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
|
||||||
/* Reset the window */
|
/* Reset the window */
|
||||||
ZSTD_window_clear(&serialState->ldmState.window);
|
ZSTD_window_init(&serialState->ldmState.window);
|
||||||
serialState->ldmWindow = serialState->ldmState.window;
|
serialState->ldmWindow = serialState->ldmState.window;
|
||||||
/* Resize tables and output space if necessary. */
|
/* Resize tables and output space if necessary. */
|
||||||
if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
|
if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
|
||||||
@ -1786,7 +1786,7 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
|
|||||||
BYTE const* const bufferStart = (BYTE const*)buffer.start;
|
BYTE const* const bufferStart = (BYTE const*)buffer.start;
|
||||||
BYTE const* const bufferEnd = bufferStart + buffer.capacity;
|
BYTE const* const bufferEnd = bufferStart + buffer.capacity;
|
||||||
BYTE const* const rangeStart = (BYTE const*)range.start;
|
BYTE const* const rangeStart = (BYTE const*)range.start;
|
||||||
BYTE const* const rangeEnd = rangeStart + range.size;
|
BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart;
|
||||||
|
|
||||||
if (rangeStart == NULL || bufferStart == NULL)
|
if (rangeStart == NULL || bufferStart == NULL)
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -181,17 +181,29 @@ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize
|
|||||||
|
|
||||||
/* fill DTable */
|
/* fill DTable */
|
||||||
{ U32 n;
|
{ U32 n;
|
||||||
for (n=0; n<nbSymbols; n++) {
|
size_t const nEnd = nbSymbols;
|
||||||
U32 const w = huffWeight[n];
|
for (n=0; n<nEnd; n++) {
|
||||||
U32 const length = (1 << w) >> 1;
|
size_t const w = huffWeight[n];
|
||||||
U32 u;
|
size_t const length = (1 << w) >> 1;
|
||||||
|
size_t const uStart = rankVal[w];
|
||||||
|
size_t const uEnd = uStart + length;
|
||||||
|
size_t u;
|
||||||
HUF_DEltX1 D;
|
HUF_DEltX1 D;
|
||||||
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
|
D.byte = (BYTE)n;
|
||||||
for (u = rankVal[w]; u < rankVal[w] + length; u++)
|
D.nbBits = (BYTE)(tableLog + 1 - w);
|
||||||
dt[u] = D;
|
rankVal[w] = (U32)uEnd;
|
||||||
rankVal[w] += length;
|
if (length < 4) {
|
||||||
} }
|
/* Use length in the loop bound so the compiler knows it is short. */
|
||||||
|
for (u = 0; u < length; ++u)
|
||||||
|
dt[uStart + u] = D;
|
||||||
|
} else {
|
||||||
|
/* Unroll the loop 4 times, we know it is a power of 2. */
|
||||||
|
for (u = uStart; u < uEnd; u += 4) {
|
||||||
|
dt[u + 0] = D;
|
||||||
|
dt[u + 1] = D;
|
||||||
|
dt[u + 2] = D;
|
||||||
|
dt[u + 3] = D;
|
||||||
|
} } } }
|
||||||
return iSize;
|
return iSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -618,7 +618,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)(*srcPtr);
|
const BYTE* ip = (const BYTE*)(*srcPtr);
|
||||||
BYTE* const ostart = (BYTE* const)dst;
|
BYTE* const ostart = (BYTE* const)dst;
|
||||||
BYTE* const oend = ostart + dstCapacity;
|
BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
|
||||||
BYTE* op = ostart;
|
BYTE* op = ostart;
|
||||||
size_t remainingSrcSize = *srcSizePtr;
|
size_t remainingSrcSize = *srcSizePtr;
|
||||||
|
|
||||||
@ -669,7 +669,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|||||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||||
if (dctx->fParams.checksumFlag)
|
if (dctx->fParams.checksumFlag)
|
||||||
XXH64_update(&dctx->xxhState, op, decodedSize);
|
XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||||
op += decodedSize;
|
if (decodedSize != 0)
|
||||||
|
op += decodedSize;
|
||||||
|
assert(ip != NULL);
|
||||||
ip += cBlockSize;
|
ip += cBlockSize;
|
||||||
remainingSrcSize -= cBlockSize;
|
remainingSrcSize -= cBlockSize;
|
||||||
if (blockProperties.lastBlock) break;
|
if (blockProperties.lastBlock) break;
|
||||||
@ -776,7 +778,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|||||||
"error.");
|
"error.");
|
||||||
if (ZSTD_isError(res)) return res;
|
if (ZSTD_isError(res)) return res;
|
||||||
assert(res <= dstCapacity);
|
assert(res <= dstCapacity);
|
||||||
dst = (BYTE*)dst + res;
|
if (res != 0)
|
||||||
|
dst = (BYTE*)dst + res;
|
||||||
dstCapacity -= res;
|
dstCapacity -= res;
|
||||||
}
|
}
|
||||||
moreThan1Frame = 1;
|
moreThan1Frame = 1;
|
||||||
@ -1486,11 +1489,13 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
|
|||||||
|
|
||||||
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
|
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
|
||||||
{
|
{
|
||||||
const char* const istart = (const char*)(input->src) + input->pos;
|
const char* const src = (const char*)input->src;
|
||||||
const char* const iend = (const char*)(input->src) + input->size;
|
const char* const istart = input->pos != 0 ? src + input->pos : src;
|
||||||
|
const char* const iend = input->size != 0 ? src + input->size : src;
|
||||||
const char* ip = istart;
|
const char* ip = istart;
|
||||||
char* const ostart = (char*)(output->dst) + output->pos;
|
char* const dst = (char*)output->dst;
|
||||||
char* const oend = (char*)(output->dst) + output->size;
|
char* const ostart = output->pos != 0 ? dst + output->pos : dst;
|
||||||
|
char* const oend = output->size != 0 ? dst + output->size : dst;
|
||||||
char* op = ostart;
|
char* op = ostart;
|
||||||
U32 someMoreWork = 1;
|
U32 someMoreWork = 1;
|
||||||
|
|
||||||
|
@ -763,7 +763,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
|||||||
|
|
||||||
/* This function is redundant with the advanced API and equivalent to:
|
/* This function is redundant with the advanced API and equivalent to:
|
||||||
*
|
*
|
||||||
* ZSTD_DCtx_reset(zds);
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
||||||
* ZSTD_DCtx_refDDict(zds, NULL);
|
* ZSTD_DCtx_refDDict(zds, NULL);
|
||||||
*/
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
||||||
|
@ -73,7 +73,7 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
|
|||||||
{
|
{
|
||||||
/* (atime, mtime) */
|
/* (atime, mtime) */
|
||||||
struct timespec timebuf[2] = { {0, UTIME_NOW} };
|
struct timespec timebuf[2] = { {0, UTIME_NOW} };
|
||||||
timebuf[1] = statbuf->st_mtim;
|
timebuf[1].tv_sec = statbuf->st_mtime;
|
||||||
res += utimensat(AT_FDCWD, filename, timebuf, 0);
|
res += utimensat(AT_FDCWD, filename, timebuf, 0);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -402,7 +402,7 @@ The list of available _options_:
|
|||||||
Bigger hash tables cause less collisions which usually makes compression
|
Bigger hash tables cause less collisions which usually makes compression
|
||||||
faster, but requires more memory during compression.
|
faster, but requires more memory during compression.
|
||||||
|
|
||||||
The minimum _hlog_ is 6 (64 B) and the maximum is 26 (128 MiB).
|
The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
|
||||||
|
|
||||||
- `chainLog`=_clog_, `clog`=_clog_:
|
- `chainLog`=_clog_, `clog`=_clog_:
|
||||||
Specify the maximum number of bits for a hash chain or a binary tree.
|
Specify the maximum number of bits for a hash chain or a binary tree.
|
||||||
@ -413,7 +413,8 @@ The list of available _options_:
|
|||||||
compression.
|
compression.
|
||||||
This option is ignored for the ZSTD_fast strategy.
|
This option is ignored for the ZSTD_fast strategy.
|
||||||
|
|
||||||
The minimum _clog_ is 6 (64 B) and the maximum is 28 (256 MiB).
|
The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
|
||||||
|
and 30 (1 Gib) on 64-bit platforms.
|
||||||
|
|
||||||
- `searchLog`=_slog_, `slog`=_slog_:
|
- `searchLog`=_slog_, `slog`=_slog_:
|
||||||
Specify the maximum number of searches in a hash chain or a binary tree
|
Specify the maximum number of searches in a hash chain or a binary tree
|
||||||
@ -422,7 +423,7 @@ The list of available _options_:
|
|||||||
More searches increases the chance to find a match which usually increases
|
More searches increases the chance to find a match which usually increases
|
||||||
compression ratio but decreases compression speed.
|
compression ratio but decreases compression speed.
|
||||||
|
|
||||||
The minimum _slog_ is 1 and the maximum is 26.
|
The minimum _slog_ is 1 and the maximum is 'windowLog' - 1.
|
||||||
|
|
||||||
- `minMatch`=_mml_, `mml`=_mml_:
|
- `minMatch`=_mml_, `mml`=_mml_:
|
||||||
Specify the minimum searched length of a match in a hash table.
|
Specify the minimum searched length of a match in a hash table.
|
||||||
@ -447,7 +448,7 @@ The list of available _options_:
|
|||||||
|
|
||||||
For all other strategies, this field has no impact.
|
For all other strategies, this field has no impact.
|
||||||
|
|
||||||
The minimum _tlen_ is 0 and the maximum is 999.
|
The minimum _tlen_ is 0 and the maximum is 128 Kib.
|
||||||
|
|
||||||
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
|
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
|
||||||
Determine `overlapSize`, amount of data reloaded from previous job.
|
Determine `overlapSize`, amount of data reloaded from previous job.
|
||||||
@ -470,7 +471,7 @@ The list of available _options_:
|
|||||||
Bigger hash tables usually improve compression ratio at the expense of more
|
Bigger hash tables usually improve compression ratio at the expense of more
|
||||||
memory during compression and a decrease in compression speed.
|
memory during compression and a decrease in compression speed.
|
||||||
|
|
||||||
The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
|
The minimum _lhlog_ is 6 and the maximum is 30 (default: 20).
|
||||||
|
|
||||||
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
|
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
|
||||||
Specify the minimum searched length of a match for long distance matching.
|
Specify the minimum searched length of a match for long distance matching.
|
||||||
@ -490,7 +491,7 @@ The list of available _options_:
|
|||||||
Larger bucket sizes improve collision resolution but decrease compression
|
Larger bucket sizes improve collision resolution but decrease compression
|
||||||
speed.
|
speed.
|
||||||
|
|
||||||
The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
|
The minimum _lblog_ is 1 and the maximum is 8 (default: 3).
|
||||||
|
|
||||||
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
|
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
|
||||||
Specify the frequency of inserting entries into the long distance matching
|
Specify the frequency of inserting entries into the long distance matching
|
||||||
|
6
tests/fuzz/.gitignore
vendored
6
tests/fuzz/.gitignore
vendored
@ -2,7 +2,13 @@
|
|||||||
corpora
|
corpora
|
||||||
block_decompress
|
block_decompress
|
||||||
block_round_trip
|
block_round_trip
|
||||||
|
dictionary_decompress
|
||||||
|
dictionary_loader
|
||||||
|
dictionary_round_trip
|
||||||
|
simple_compress
|
||||||
simple_decompress
|
simple_decompress
|
||||||
simple_round_trip
|
simple_round_trip
|
||||||
stream_decompress
|
stream_decompress
|
||||||
stream_round_trip
|
stream_round_trip
|
||||||
|
zstd_frame_info
|
||||||
|
fuzz-*.log
|
@ -502,6 +502,19 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
|
||||||
|
{ int level = -1;
|
||||||
|
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||||
|
if (!cctx) goto _output_error;
|
||||||
|
for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
|
||||||
|
CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
|
||||||
|
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
|
||||||
|
CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
|
||||||
|
}
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
|
DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
|
||||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
|
size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
|
||||||
@ -2057,14 +2070,14 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||||||
|
|
||||||
/* long rle test */
|
/* long rle test */
|
||||||
{ size_t sampleSize = 0;
|
{ size_t sampleSize = 0;
|
||||||
|
size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
|
||||||
DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
|
||||||
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
|
|
||||||
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
|
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
|
||||||
sampleSize += 256 KB - 1;
|
sampleSize += 256 KB - 1;
|
||||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
|
memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
|
||||||
sampleSize += 96 KB;
|
sampleSize += 96 KB;
|
||||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||||
if (ZSTD_isError(cSize)) goto _output_error;
|
if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
|
||||||
{ CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
{ CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
||||||
if (regenSize!=sampleSize) goto _output_error; }
|
if (regenSize!=sampleSize) goto _output_error; }
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
@ -509,6 +509,33 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
|
||||||
|
inBuff.src = NULL;
|
||||||
|
inBuff.size = 0;
|
||||||
|
inBuff.pos = 0;
|
||||||
|
outBuff.dst = NULL;
|
||||||
|
outBuff.size = 0;
|
||||||
|
outBuff.pos = 0;
|
||||||
|
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||||
|
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
|
||||||
|
CHECK_Z( ZSTD_endStream(zc, &outBuff) );
|
||||||
|
outBuff.dst = (char*)(compressedBuffer);
|
||||||
|
outBuff.size = compressedBufferSize;
|
||||||
|
outBuff.pos = 0;
|
||||||
|
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||||
|
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
|
||||||
|
}
|
||||||
|
inBuff.src = outBuff.dst;
|
||||||
|
inBuff.size = outBuff.pos;
|
||||||
|
inBuff.pos = 0;
|
||||||
|
outBuff.dst = NULL;
|
||||||
|
outBuff.size = 0;
|
||||||
|
outBuff.pos = 0;
|
||||||
|
CHECK_Z( ZSTD_initDStream(zd) );
|
||||||
|
{ size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||||
|
if (ret != 0) goto _output_error;
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK\n");
|
||||||
/* _srcSize compression test */
|
/* _srcSize compression test */
|
||||||
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||||
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
|
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
|
||||||
|
Loading…
Reference in New Issue
Block a user