AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Changed ZSTD_compressEnd()

Browse Source
This commit is contained in:
Yann Collet 2016-07-28 15:29:08 +02:00
parent e7bf9156d1
commit 62470b4bab
6 changed files with 36 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS
View File

@ -1,6 +1,6 @@
v0.8.0
New : updated compresson format
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
Changed : modified API : ZSTD_compressEnd()
Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
Fixed : statistics for large dictionaries (> 128 KB), reported by Ilona Papava

2
lib/compress/zbuff_compress.c
View File

@ -306,7 +306,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
}
/* create epilogue */
zbc->stage = ZBUFFcs_final;
zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* epilogue into outBuff */
}
/* flush epilogue */

56
lib/compress/zstd_compress.c
View File

@ -2287,7 +2287,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
op += cSize;
}
if (lastFrameChunk) cctx->stage = ZSTDcs_ending;
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); /* debug only */
return op-ostart;
}
@ -2401,19 +2401,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
}
size_t ZSTD_compressContinueThenEnd (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t endResult;
size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
if (ZSTD_isError(cSize)) return cSize;
endResult = ZSTD_compressEnd(cctx, (char*)dst + cSize, dstCapacity-cSize);
if (ZSTD_isError(endResult)) return endResult;
return cSize + endResult;
}
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx)
{
return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog);
@ -2593,10 +2580,10 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
}
/*! ZSTD_compressEnd() :
* Write frame epilogue.
/*! ZSTD_writeEpilogue() :
* Ends a frame.
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
@ -2634,6 +2621,19 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
}
size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t endResult;
size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
if (ZSTD_isError(cSize)) return cSize;
endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
if (ZSTD_isError(endResult)) return endResult;
return cSize + endResult;
}
/*! ZSTD_compress_usingPreparedCCtx() :
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
@ -2643,12 +2643,10 @@ static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx*
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
{ size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx);
if (ZSTD_isError(errorCode)) return errorCode;
}
{ size_t const cSize = ZSTD_compressContinueThenEnd(cctx, dst, dstCapacity, src, srcSize);
return cSize;
}
size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx);
if (ZSTD_isError(errorCode)) return errorCode;
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
@ -2658,16 +2656,10 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
const void* dict,size_t dictSize,
ZSTD_parameters params)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize);
if(ZSTD_isError(errorCode)) return errorCode;
/* Init */
{ size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize);
if(ZSTD_isError(errorCode)) return errorCode; }
/* body (compression) */
{ size_t const oSize = ZSTD_compressContinueThenEnd(cctx, op, dstCapacity, src, srcSize);
return oSize; }
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,

13
lib/zstd.h
View File

@ -315,15 +315,14 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with a lot of restrictions (documented below).
* For an easier streaming API, look into common/zbuff.h
* which removes all restrictions by implementing and managing its own internal buffer */
* which removes all restrictions by allocating and managing its own internal buffer */
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
ZSTDLIB_API size_t ZSTD_compressContinueThenEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*
A ZSTD_CCtx object is required to track streaming operations.
@ -338,7 +337,7 @@ ZSTDLIB_API size_t ZSTD_compressContinueThenEnd(ZSTD_CCtx* cctx, void* dst, size
Then, consume your input using ZSTD_compressContinue().
There are some important considerations to keep in mind when using this advanced function :
- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
- Interface is synchronous : input is consumed entirely and produce 1 (or more) compressed blocks.
- Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
@ -347,9 +346,9 @@ ZSTDLIB_API size_t ZSTD_compressContinueThenEnd(ZSTD_CCtx* cctx, void* dst, size
- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the epilogue,
or ZSTD_compressContinueThenEnd(), which will write the last block.
Without last block / epilogue mark, frames will be considered unfinished (broken) by decoders.
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
Without last block mark, frames will be considered unfinished (broken) by decoders.
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
*/

5
programs/fullbench.c
View File

@ -173,12 +173,9 @@ static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2,
static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
size_t compressedSize;
(void)buff2;
ZSTD_compressBegin(g_zcc, 1);
compressedSize = ZSTD_compressContinue(g_zcc, dst, dstCapacity, src, srcSize);
compressedSize += ZSTD_compressEnd(g_zcc, ((char*)dst)+compressedSize, dstCapacity-compressedSize);
return compressedSize;
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)

10
programs/fuzzer.c
View File

@ -186,11 +186,9 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
cSize = 0;
CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
cSize += r);
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
cSize += r);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
@ -204,11 +202,9 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
{ size_t const cSizeOrig = cSize;
cSize = 0;
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
cSize += r);
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
cSize += r);
if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
@ -696,7 +692,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
totalTestSize += segmentSize;
} }
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult;
}