Merge branch 'dev' into oss

This commit is contained in:
Bimba Shrestha 2019-11-22 17:29:33 -08:00 committed by GitHub
commit 826b555463
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 137 additions and 51 deletions

3
.gitignore vendored
View File

@ -31,6 +31,7 @@ projects/
bin/ bin/
.buckd/ .buckd/
buck-out/ buck-out/
build-*
# Other files # Other files
.directory .directory
@ -43,3 +44,5 @@ _zstdbench/
googletest/ googletest/
*.d *.d
*.vscode *.vscode
compile_commands.json
.clangd

View File

@ -49,6 +49,9 @@ target_link_libraries(fullbench libzstd_static)
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c) add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
target_link_libraries(fuzzer libzstd_static) target_link_libraries(fuzzer libzstd_static)
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
target_link_libraries(zstreamtest libzstd_static)
if (UNIX) if (UNIX)
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c) add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
target_link_libraries(paramgrill libzstd_static m) #m is math library target_link_libraries(paramgrill libzstd_static m) #m is math library

View File

@ -161,7 +161,7 @@ ifneq (,$(filter Windows%,$(OS)))
LIBZSTD = dll\libzstd.dll LIBZSTD = dll\libzstd.dll
$(LIBZSTD): $(ZSTD_FILES) $(LIBZSTD): $(ZSTD_FILES)
@echo compiling dynamic library $(LIBVER) @echo compiling dynamic library $(LIBVER)
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@ $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@
else else

View File

@ -1336,10 +1336,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
if (forceResetIndex == ZSTDirp_reset) { if (forceResetIndex == ZSTDirp_reset) {
memset(&ms->window, 0, sizeof(ms->window)); ZSTD_window_init(&ms->window);
ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */
ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */
ZSTD_cwksp_mark_tables_dirty(ws); ZSTD_cwksp_mark_tables_dirty(ws);
} }
@ -1432,7 +1429,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const matchStateSize = ZSTD_sizeof_matchState(&params.cParams, /* forCCtx */ 1); size_t const matchStateSize = ZSTD_sizeof_matchState(&params.cParams, /* forCCtx */ 1);
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue; ZSTD_indexResetPolicy_e needsIndexReset = zc->initialized ? ZSTDirp_continue : ZSTDirp_reset;
if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) {
needsIndexReset = ZSTDirp_reset; needsIndexReset = ZSTDirp_reset;
@ -1557,11 +1554,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
zc->maxNbLdmSequences = maxNbLdmSeq; zc->maxNbLdmSequences = maxNbLdmSeq;
memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); ZSTD_window_init(&zc->ldmState.window);
ZSTD_window_clear(&zc->ldmState.window); ZSTD_window_clear(&zc->ldmState.window);
} }
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
zc->initialized = 1;
return 0; return 0;
} }
@ -2593,23 +2591,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
/* Ensure hash/chain table insertion resumes no sooner than lowlimit */ /* Ensure hash/chain table insertion resumes no sooner than lowlimit */
if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
{ int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams); { size_t cSize;
size_t cSize = 0; int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams);
if (useTargetCBlockSize) { if (useTargetCBlockSize) {
cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize); FORWARD_IF_ERROR(cSize);
} else { } else {
cSize = ZSTD_compressBlock_internal(cctx, cSize = ZSTD_compressBlock_internal(cctx,
op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
ip, blockSize, 0 /* frame */); ip, blockSize, 1 /* frame */);
FORWARD_IF_ERROR(cSize); FORWARD_IF_ERROR(cSize);
if (cSize == 0) { /* block is not compressible */ if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize); FORWARD_IF_ERROR(cSize);
} else { } else {
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); U32 const cBlockHeader = cSize == 1 ?
MEM_writeLE24(op, cBlockHeader24); lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader);
cSize += ZSTD_blockHeaderSize; cSize += ZSTD_blockHeaderSize;
} }
} }
@ -3767,11 +3767,11 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_EndDirective const flushMode) ZSTD_EndDirective const flushMode)
{ {
const char* const istart = (const char*)input->src; const char* const istart = (const char*)input->src;
const char* const iend = istart + input->size; const char* const iend = input->size != 0 ? istart + input->size : istart;
const char* ip = istart + input->pos; const char* ip = input->pos != 0 ? istart + input->pos : istart;
char* const ostart = (char*)output->dst; char* const ostart = (char*)output->dst;
char* const oend = ostart + output->size; char* const oend = output->size != 0 ? ostart + output->size : ostart;
char* op = ostart + output->pos; char* op = output->pos != 0 ? ostart + output->pos : ostart;
U32 someMoreWork = 1; U32 someMoreWork = 1;
/* check expectations */ /* check expectations */
@ -3810,7 +3810,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
zcs->inBuff + zcs->inBuffPos, toLoad, zcs->inBuff + zcs->inBuffPos, toLoad,
ip, iend-ip); ip, iend-ip);
zcs->inBuffPos += loaded; zcs->inBuffPos += loaded;
ip += loaded; if (loaded != 0)
ip += loaded;
if ( (flushMode == ZSTD_e_continue) if ( (flushMode == ZSTD_e_continue)
&& (zcs->inBuffPos < zcs->inBuffTarget) ) { && (zcs->inBuffPos < zcs->inBuffTarget) ) {
/* not enough input to fill full block : stop here */ /* not enough input to fill full block : stop here */
@ -3870,7 +3871,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
zcs->outBuff + zcs->outBuffFlushedSize, toFlush); zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
(unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
op += flushed; if (flushed)
op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { if (toFlush!=flushed) {
/* flush not fully completed, presumably because dst is too small */ /* flush not fully completed, presumably because dst is too small */

View File

@ -249,6 +249,7 @@ struct ZSTD_CCtx_s {
size_t staticSize; size_t staticSize;
SeqCollector seqCollector; SeqCollector seqCollector;
int isFirstBlock; int isFirstBlock;
int initialized;
seqStore_t seqStore; /* sequences storage ptrs */ seqStore_t seqStore; /* sequences storage ptrs */
ldmState_t ldmState; /* long distance matching state */ ldmState_t ldmState; /* long distance matching state */
@ -858,6 +859,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
} } } } } }
} }
MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
memset(window, 0, sizeof(*window));
window->base = (BYTE const*)"";
window->dictBase = (BYTE const*)"";
window->dictLimit = 1; /* start from 1, so that 1st position is valid */
window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
window->nextSrc = window->base + 1; /* see issue #1241 */
}
/** /**
* ZSTD_window_update(): * ZSTD_window_update():
* Updates the window by appending [src, src + srcSize) to the window. * Updates the window by appending [src, src + srcSize) to the window.
@ -871,6 +881,10 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
BYTE const* const ip = (BYTE const*)src; BYTE const* const ip = (BYTE const*)src;
U32 contiguous = 1; U32 contiguous = 1;
DEBUGLOG(5, "ZSTD_window_update"); DEBUGLOG(5, "ZSTD_window_update");
if (srcSize == 0)
return contiguous;
assert(window->base != NULL);
assert(window->dictBase != NULL);
/* Check if blocks follow each other */ /* Check if blocks follow each other */
if (src != window->nextSrc) { if (src != window->nextSrc) {
/* not contiguous */ /* not contiguous */

View File

@ -96,7 +96,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
dictCParams->hashLog : hBitsL; dictCParams->hashLog : hBitsL;
const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ? const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ?
dictCParams->chainLog : hBitsS; dictCParams->chainLog : hBitsS;
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart); const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic");
@ -271,7 +271,7 @@ _match_stored:
U32 const repIndex2 = current2 - offset_2; U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState
&& repIndex2 < prefixLowestIndex ? && repIndex2 < prefixLowestIndex ?
dictBase - dictIndexDelta + repIndex2 : dictBase + repIndex2 - dictIndexDelta :
base + repIndex2; base + repIndex2;
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) { && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {

View File

@ -660,7 +660,7 @@ ZSTD_compressBlock_lazy_generic(
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
prefixLowestIndex - (U32)(dictEnd - dictBase) : prefixLowestIndex - (U32)(dictEnd - dictBase) :
0; 0;
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest); const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
/* init */ /* init */
ip += (dictAndPrefixLength == 0); ip += (dictAndPrefixLength == 0);

View File

@ -490,7 +490,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
/* Size the seq pool tables */ /* Size the seq pool tables */
ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
/* Reset the window */ /* Reset the window */
ZSTD_window_clear(&serialState->ldmState.window); ZSTD_window_init(&serialState->ldmState.window);
serialState->ldmWindow = serialState->ldmState.window; serialState->ldmWindow = serialState->ldmState.window;
/* Resize tables and output space if necessary. */ /* Resize tables and output space if necessary. */
if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
@ -1786,7 +1786,7 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
BYTE const* const bufferStart = (BYTE const*)buffer.start; BYTE const* const bufferStart = (BYTE const*)buffer.start;
BYTE const* const bufferEnd = bufferStart + buffer.capacity; BYTE const* const bufferEnd = bufferStart + buffer.capacity;
BYTE const* const rangeStart = (BYTE const*)range.start; BYTE const* const rangeStart = (BYTE const*)range.start;
BYTE const* const rangeEnd = rangeStart + range.size; BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart;
if (rangeStart == NULL || bufferStart == NULL) if (rangeStart == NULL || bufferStart == NULL)
return 0; return 0;

View File

@ -181,17 +181,29 @@ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize
/* fill DTable */ /* fill DTable */
{ U32 n; { U32 n;
for (n=0; n<nbSymbols; n++) { size_t const nEnd = nbSymbols;
U32 const w = huffWeight[n]; for (n=0; n<nEnd; n++) {
U32 const length = (1 << w) >> 1; size_t const w = huffWeight[n];
U32 u; size_t const length = (1 << w) >> 1;
size_t const uStart = rankVal[w];
size_t const uEnd = uStart + length;
size_t u;
HUF_DEltX1 D; HUF_DEltX1 D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); D.byte = (BYTE)n;
for (u = rankVal[w]; u < rankVal[w] + length; u++) D.nbBits = (BYTE)(tableLog + 1 - w);
dt[u] = D; rankVal[w] = (U32)uEnd;
rankVal[w] += length; if (length < 4) {
} } /* Use length in the loop bound so the compiler knows it is short. */
for (u = 0; u < length; ++u)
dt[uStart + u] = D;
} else {
/* Unroll the loop 4 times, we know it is a power of 2. */
for (u = uStart; u < uEnd; u += 4) {
dt[u + 0] = D;
dt[u + 1] = D;
dt[u + 2] = D;
dt[u + 3] = D;
} } } }
return iSize; return iSize;
} }

View File

@ -618,7 +618,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
{ {
const BYTE* ip = (const BYTE*)(*srcPtr); const BYTE* ip = (const BYTE*)(*srcPtr);
BYTE* const ostart = (BYTE* const)dst; BYTE* const ostart = (BYTE* const)dst;
BYTE* const oend = ostart + dstCapacity; BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
BYTE* op = ostart; BYTE* op = ostart;
size_t remainingSrcSize = *srcSizePtr; size_t remainingSrcSize = *srcSizePtr;
@ -669,7 +669,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
if (dctx->fParams.checksumFlag) if (dctx->fParams.checksumFlag)
XXH64_update(&dctx->xxhState, op, decodedSize); XXH64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize; if (decodedSize != 0)
op += decodedSize;
assert(ip != NULL);
ip += cBlockSize; ip += cBlockSize;
remainingSrcSize -= cBlockSize; remainingSrcSize -= cBlockSize;
if (blockProperties.lastBlock) break; if (blockProperties.lastBlock) break;
@ -776,7 +778,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
"error."); "error.");
if (ZSTD_isError(res)) return res; if (ZSTD_isError(res)) return res;
assert(res <= dstCapacity); assert(res <= dstCapacity);
dst = (BYTE*)dst + res; if (res != 0)
dst = (BYTE*)dst + res;
dstCapacity -= res; dstCapacity -= res;
} }
moreThan1Frame = 1; moreThan1Frame = 1;
@ -1486,11 +1489,13 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{ {
const char* const istart = (const char*)(input->src) + input->pos; const char* const src = (const char*)input->src;
const char* const iend = (const char*)(input->src) + input->size; const char* const istart = input->pos != 0 ? src + input->pos : src;
const char* const iend = input->size != 0 ? src + input->size : src;
const char* ip = istart; const char* ip = istart;
char* const ostart = (char*)(output->dst) + output->pos; char* const dst = (char*)output->dst;
char* const oend = (char*)(output->dst) + output->size; char* const ostart = output->pos != 0 ? dst + output->pos : dst;
char* const oend = output->size != 0 ? dst + output->size : dst;
char* op = ostart; char* op = ostart;
U32 someMoreWork = 1; U32 someMoreWork = 1;

View File

@ -763,7 +763,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
/* This function is redundant with the advanced API and equivalent to: /* This function is redundant with the advanced API and equivalent to:
* *
* ZSTD_DCtx_reset(zds); * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
* ZSTD_DCtx_refDDict(zds, NULL); * ZSTD_DCtx_refDDict(zds, NULL);
*/ */
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);

View File

@ -73,7 +73,7 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{ {
/* (atime, mtime) */ /* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW} }; struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim; timebuf[1].tv_sec = statbuf->st_mtime;
res += utimensat(AT_FDCWD, filename, timebuf, 0); res += utimensat(AT_FDCWD, filename, timebuf, 0);
} }
#endif #endif

View File

@ -402,7 +402,7 @@ The list of available _options_:
Bigger hash tables cause less collisions which usually makes compression Bigger hash tables cause less collisions which usually makes compression
faster, but requires more memory during compression. faster, but requires more memory during compression.
The minimum _hlog_ is 6 (64 B) and the maximum is 26 (128 MiB). The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
- `chainLog`=_clog_, `clog`=_clog_: - `chainLog`=_clog_, `clog`=_clog_:
Specify the maximum number of bits for a hash chain or a binary tree. Specify the maximum number of bits for a hash chain or a binary tree.
@ -413,7 +413,8 @@ The list of available _options_:
compression. compression.
This option is ignored for the ZSTD_fast strategy. This option is ignored for the ZSTD_fast strategy.
The minimum _clog_ is 6 (64 B) and the maximum is 28 (256 MiB). The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
and 30 (1 Gib) on 64-bit platforms.
- `searchLog`=_slog_, `slog`=_slog_: - `searchLog`=_slog_, `slog`=_slog_:
Specify the maximum number of searches in a hash chain or a binary tree Specify the maximum number of searches in a hash chain or a binary tree
@ -422,7 +423,7 @@ The list of available _options_:
More searches increases the chance to find a match which usually increases More searches increases the chance to find a match which usually increases
compression ratio but decreases compression speed. compression ratio but decreases compression speed.
The minimum _slog_ is 1 and the maximum is 26. The minimum _slog_ is 1 and the maximum is 'windowLog' - 1.
- `minMatch`=_mml_, `mml`=_mml_: - `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table. Specify the minimum searched length of a match in a hash table.
@ -447,7 +448,7 @@ The list of available _options_:
For all other strategies, this field has no impact. For all other strategies, this field has no impact.
The minimum _tlen_ is 0 and the maximum is 999. The minimum _tlen_ is 0 and the maximum is 128 Kib.
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_: - `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
Determine `overlapSize`, amount of data reloaded from previous job. Determine `overlapSize`, amount of data reloaded from previous job.
@ -470,7 +471,7 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed. memory during compression and a decrease in compression speed.
The minimum _lhlog_ is 6 and the maximum is 26 (default: 20). The minimum _lhlog_ is 6 and the maximum is 30 (default: 20).
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_: - `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching. Specify the minimum searched length of a match for long distance matching.
@ -490,7 +491,7 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression Larger bucket sizes improve collision resolution but decrease compression
speed. speed.
The minimum _lblog_ is 0 and the maximum is 8 (default: 3). The minimum _lblog_ is 1 and the maximum is 8 (default: 3).
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_: - `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching Specify the frequency of inserting entries into the long distance matching

View File

@ -2,7 +2,13 @@
corpora corpora
block_decompress block_decompress
block_round_trip block_round_trip
dictionary_decompress
dictionary_loader
dictionary_round_trip
simple_compress
simple_decompress simple_decompress
simple_round_trip simple_round_trip
stream_decompress stream_decompress
stream_round_trip stream_round_trip
zstd_frame_info
fuzz-*.log

View File

@ -502,6 +502,19 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
{ int level = -1;
ZSTD_CCtx* cctx = ZSTD_createCCtx();
if (!cctx) goto _output_error;
for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx(); { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
@ -2057,14 +2070,14 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* long rle test */ /* long rle test */
{ size_t sampleSize = 0; { size_t sampleSize = 0;
size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1; sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2); memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
sampleSize += 96 KB; sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
{ CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
if (regenSize!=sampleSize) goto _output_error; } if (regenSize!=sampleSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");

View File

@ -509,6 +509,33 @@ static int basicUnitTests(U32 seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
inBuff.src = NULL;
inBuff.size = 0;
inBuff.pos = 0;
outBuff.dst = NULL;
outBuff.size = 0;
outBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
CHECK_Z( ZSTD_endStream(zc, &outBuff) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
inBuff.src = outBuff.dst;
inBuff.size = outBuff.pos;
inBuff.pos = 0;
outBuff.dst = NULL;
outBuff.size = 0;
outBuff.pos = 0;
CHECK_Z( ZSTD_initDStream(zd) );
{ size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ret != 0) goto _output_error;
}
DISPLAYLEVEL(3, "OK\n");
/* _srcSize compression test */ /* _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );