AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added test case

Browse Source
This commit is contained in:
Yann Collet 2018-06-05 14:53:28 -07:00
parent 3d523c741b
commit f7392f3dc9
3 changed files with 43 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/zstd_manual.html
View File

@ -82,7 +82,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
</b><p> `src` should point to the start of a ZSTD encoded frame.
`srcSize` must be at least as large as the frame header.
hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
@return : - decompressed size of the frame in `src`, if known
@return : - decompressed size of `src` frame content, if known
- ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
- ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
note 1 : a 0 return value means the frame is valid but "empty".
@ -92,7 +92,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
Optionally, application can rely on some implicit limit,
as ZSTD_decompress() only needs an upper bound of decompressed size.
(For example, data could be necessarily cut into blocks <= 16 KB).
note 3 : decompressed size is always present when compression is done with ZSTD_compress()
note 3 : decompressed size is always present when compression is completed using single-pass functions,
such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
note 4 : decompressed size can be very large (64-bits value),
potentially larger than what local system can handle as a single memory segment.
In which case, it's necessary to use streaming mode to decompress data.
@ -107,8 +108,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
Both functions work the same way, but ZSTD_getDecompressedSize() blends
"empty", "unknown" and "error" results to the same return value (0),
while ZSTD_getFrameContentSize() gives them separate return values.
`src` is the start of a zstd compressed frame.
@return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise.
@return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise.
</p></pre><BR>
<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>

6
lib/compress/zstd_compress.c
View File

@ -1079,6 +1079,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zbuff, pledgedSrcSize)) {
DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
zc->appliedParams.cParams.windowLog, zc->blockSize);
zc->workSpaceTooLarge += (zc->workSpaceTooLarge > 0); /* if it was too large, it still is */
return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
} }
DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
@ -1116,12 +1117,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
int const workSpaceTooSmall = zc->workSpaceSize < neededSpace;
int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace;
int const workSpaceWasteful = 0 && workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",
neededSpace>>10, matchStateSize>>10, bufferSpace>>10);
DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
if (workSpaceTooSmall || workSpaceWasteful) {
DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB",
zc->workSpaceSize >> 10,

35
tests/fuzzer.c
View File

@ -450,6 +450,7 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
/* this test is really too long, and should be made faster */
DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
@ -467,6 +468,40 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
{ ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
assert(largeCCtx != NULL);
CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
{ size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
{ ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
assert(smallCCtx != NULL);
CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
{ size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
largeCCtxSize>>10, smallCCtxSize>>10);
assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
* make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
}
ZSTD_freeCCtx(smallCCtx);
}
{ U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
* currently defined as 128x, but could be adjusted in the future.
* make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
U32 u;
for (u=0; u<maxNbAttempts; u++) {
size_t const srcSize = (FUZ_rand(&seed) & 4095) + 200;
CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize, -9));
if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
}
DISPLAYLEVEL(5, "size down after %u attempts : ", u);
if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
}
}
ZSTD_freeCCtx(largeCCtx);
}
DISPLAYLEVEL(3, "OK \n");
/* Static CCtx tests */
#define STATIC_CCTX_LEVEL 3
DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);