AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

small improvements to lz4frame compression

Browse Source
This commit is contained in:
Yann Collet 2014-09-15 00:59:30 +01:00
parent 562b34f660
commit d71b9e25b7
4 changed files with 90 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
NEWS
View File

@ -1,3 +1,7 @@
r123:
Added : experimental lz4frame API, thanks to Takayuki Matsuoka and Christopher Jackson for testings
Fix : test mode (-t) no longer requires confirmation, thanks to Thary Nguyen
r122:
Fix : AIX & AIX64 support (SamG)
Fix : mips 64-bits support (lew van)

154
lz4frame.c
View File

@ -245,6 +245,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
cctxI.maxBufferSize = 64 KB; /* mess with real buffer size, to prevent allocation; works because autoflush==1 & stableSrc==1 */
prefs.autoFlush = 1;
options.stableSrc = 1;
if (srcSize <= 64 KB) prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */
if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs))
return -ERROR_dstMaxSize_tooSmall;
@ -385,6 +386,27 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
}
typedef int (*compressFunc_t)(void*, const char*, char*, int, int);
static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx)
{
/* compress one block */
BYTE* cSizePtr = (BYTE*)dst;
U32 cSize;
cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1));
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) /* compression failed */
{
cSize = srcSize;
LZ4F_writeLE32(cSizePtr, srcSize + LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+4, src, srcSize);
}
return cSize + 4;
}
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
/* LZ4F_compressUpdate()
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
@ -403,8 +425,8 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
const BYTE* const srcEnd = srcPtr + srcSize;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
U32 lastBlockCompressed = 0;
int (*compress)(void*, const char*, char*, int, int);
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
compressFunc_t compress;
if (cctxPtr->cStage != 1) return -ERROR_GENERIC;
@ -426,26 +448,17 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
srcPtr = srcEnd;
cctxPtr->tmpInSize += srcSize;
/* still needs some CRC */
}
else
{
/* complete tmpIn block and then compress it */
BYTE* cSizePtr = dstPtr;
U32 cSize;
lastBlockCompressed = 1;
lastBlockCompressed = fromTmpBuffer;
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;
dstPtr += 4; /* space for cSize */
cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1));
dstPtr += cSize;
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) /* compression failed : non compressible assumed */
{
cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
LZ4F_writeLE32(cSizePtr, cSize);
memcpy(dstPtr, cctxPtr->tmpIn, blockSize);
dstPtr += blockSize;
}
dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, &(cctxPtr->lz4ctx));
if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
}
@ -453,59 +466,45 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
while ((size_t)(srcEnd - srcPtr) >= blockSize)
{
/* compress one block */
BYTE* cSizePtr = dstPtr;
U32 cSize;
lastBlockCompressed = 2;
dstPtr += 4; /* space for cSizePtr */
cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1));
dstPtr += cSize;
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) /* compression failed */
{
cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
LZ4F_writeLE32(cSizePtr, cSize);
memcpy(dstPtr, srcPtr, blockSize);
dstPtr += blockSize;
}
/* compress full block */
lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, &(cctxPtr->lz4ctx));
srcPtr += blockSize;
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd))
{
/* compress remaining input */
BYTE* cSizePtr = dstPtr;
U32 cSize;
size_t iSize = srcEnd - srcPtr;
lastBlockCompressed = 2;
dstPtr += 4; /* space for cSizePtr */
cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(iSize), (int)(iSize-1));
dstPtr += cSize;
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) /* compression failed */
{
cSize = iSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
LZ4F_writeLE32(cSizePtr, cSize);
memcpy(dstPtr, srcPtr, iSize);
dstPtr += iSize;
}
srcPtr += iSize;
/* compress remaining input < blockSize */
lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, &(cctxPtr->lz4ctx));
srcPtr = srcEnd;
}
/* save last input up to 64 KB for dictionary */
if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed) && (!compressOptionsPtr->stableSrc))
/* preserve dictionary if necessary */
if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer))
{
if ((lastBlockCompressed==2) ||
((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize)))
if (compressOptionsPtr->stableSrc)
{
int result;
result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
if (result==0) return -ERROR_GENERIC;
cctxPtr->tmpIn = cctxPtr->tmpBuff + result;
cctxPtr->tmpIn = cctxPtr->tmpBuff;
}
else
{
int realDictSize;
realDictSize = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
if (realDictSize==0) return -ERROR_GENERIC;
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
}
if (srcPtr < srcEnd) /* some input data left, necessarily < blockSize */
/* keep tmpIn within limits */
if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */
{
LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB;
}
/* some input data left, necessarily < blockSize */
if (srcPtr < srcEnd)
{
/* fill tmp buffer */
size_t sizeToCopy = srcEnd - srcPtr;
@ -547,31 +546,16 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer,
(int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue :
LZ4_compress_limitedOutput_withState;
{
BYTE* cSizePtr = dstPtr;
U32 cSize;
dstPtr += 4; /* space for cSizePtr */
cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(cctxPtr->tmpInSize), (int)(cctxPtr->tmpInSize-1));
dstPtr += cSize;
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) /* compression failed */
{
cSize = cctxPtr->tmpInSize + LZ4F_BLOCKUNCOMPRESSED_FLAG;
LZ4F_writeLE32(cSizePtr, cSize);
memcpy(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize);
dstPtr += cctxPtr->tmpInSize;
}
/* compress tmp buffer */
dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, &(cctxPtr->lz4ctx));
if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
cctxPtr->tmpInSize = 0;
}
if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked)
&& ((cctxPtr->tmpBuff + cctxPtr->maxBufferSize) < (cctxPtr->tmpIn + cctxPtr->maxBlockSize)))
/* keep tmpIn within limits */
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */
{
/* last 64 KB of input become dictionary */
int result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
if (!result) return ERROR_GENERIC;
cctxPtr->tmpIn = cctxPtr->tmpBuff + result;
LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB);
cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB;
}
return dstPtr - dstStart;
@ -1114,8 +1098,20 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
}
}
if ((dctxPtr->frameInfo.blockMode==blockLinked) && (dctxPtr->dict != dctxPtr->tmpOutBuffer))
if ( (dctxPtr->frameInfo.blockMode==blockLinked)
&&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
)
LZ4F_saveDict(dctxPtr, NULL, 0);
//(!decompressOptionsPtr->stableDst + 1) )
/*{
size_t newDictSize = dctxPtr->dictSize;
BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
if ((newDictSize) > 64 KB) newDictSize = 64 KB;
memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
dctxPtr->dict = dctxPtr->tmpOutBuffer;
dctxPtr->dictSize = newDictSize;
dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
}*/
if (srcPtr<srcEnd) /* function must be called again with following source data */
dctxPtr->srcExpect = srcPtr;

2
lz4frame.h
View File

@ -187,7 +187,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
* compressionContext can then be used again, starting with LZ4F_compressBegin().
*/

14
programs/frametest.c
View File

@ -406,7 +406,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
unsigned CCflag = FUZ_rand(&randState) & 1;
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
LZ4F_preferences_t prefs = { 0 };
LZ4F_compressOptions_t options = { 0 };
LZ4F_compressOptions_t cOptions = { 0 };
LZ4F_decompressOptions_t dOptions = { 0 };
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
@ -437,21 +438,21 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
size_t oSize = oend-op;
unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
options.stableSrc = ((FUZ_rand(&randState) && 3) == 2);
cOptions.stableSrc = ((FUZ_rand(&randState) && 3) == 1);
result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &options);
result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
op += result;
ip += iSize;
if (forceFlush)
{
result = LZ4F_flush(cCtx, op, oend-op, &options);
result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
op += result;
}
}
result = LZ4F_compressEnd(cCtx, op, oend-op, &options);
result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
op += result;
cSize = op-(BYTE*)compressedBuffer;
@ -471,7 +472,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
if (oSize > (size_t)(oend-op)) oSize = oend-op;
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
dOptions.stableDst = FUZ_rand(&randState) & 1;
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
CHECK(LZ4F_isError(result), "Decompression failed (error %i)", (int)result);
op += oSize;