AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

streaming API : Improved ring buffer management

Browse Source
This commit is contained in:
Yann Collet 2015-04-24 10:15:12 +01:00
parent bda38fdcb1
commit d6dc0a410d
5 changed files with 124 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/lz4.c
View File

@ -1023,8 +1023,7 @@ FORCE_INLINE int LZ4_decompress_generic(
{ {
/* match can be copied as a single segment from external dictionary */ /* match can be copied as a single segment from external dictionary */
match = dictEnd - (lowPrefix-match); match = dictEnd - (lowPrefix-match);
memcpy(op, match, length); memmove(op, match, length); op += length;
op += length;
} }
else else
{ {

39
lib/lz4.h
View File

@ -74,15 +74,14 @@ int LZ4_compress_safe (const char* source, char* dest, int sourceSize, int max
int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/* /*
LZ4_compress_limitedOutput() : LZ4_compress_safe() :
Compresses 'sourceSize' bytes from buffer 'source' Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' of size 'maxOutputSize'. into already allocated 'dest' buffer of size 'maxDestSize'.
Compression runs faster when 'maxOutputSize' >= LZ4_compressBound(sourceSize). Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
That's because in such case, it is guaranteed to compress within 'dest' budget, even in worst case scenario. It also runs faster, so it's a recommended setting.
Compressing into a more limited space budget requires additional checks. If the function cannot compress 'source' into a more limited 'dest' budget,
If the function cannot compress 'source' into a limited 'dest' budget, compression stops *immediately*, and the function result is zero.
compression stops *immediately*, and result of the function is zero. As a consequence, 'dest' content is not valid.
It greatly accelerates behavior on non-compressible input, but as a consequence, 'dest' content is not valid either.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer. This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
@ -132,13 +131,13 @@ int LZ4_compress_safe_extState (void* state, const char* source, char* dest, int
LZ4_compress_fast() : LZ4_compress_fast() :
Same as LZ4_compress_safe(), but allows to select an "acceleration" factor. Same as LZ4_compress_safe(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression. The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off, which can be fine tuned, selecting whichever value you want. It's a trade-off. It can be fine tuned, with each successive value providing an additional +2/3% to speed.
An acceleration value of "0" means "use Default value", which is typically 17 (see lz4.c source code). An acceleration value of "0" means "use Default value", which is typically 17 (see lz4.c source code).
An acceleration value of "1" is the same as regular LZ4_compress_safe() An acceleration value of "1" is the same as regular LZ4_compress_safe()
Note : this function is "safe", even if its name does not explicitly contain the word. It's just faster and compress less. Note : this function is "safe", even if its name does not explicitly contain the word. It's just faster and compress less.
*/ */
int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxOutputSize, unsigned acceleration); int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, unsigned acceleration);
int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, unsigned acceleration); int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, unsigned acceleration);
/* /*
@ -260,8 +259,18 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
*_continue() : *_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode. These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
If this condition is not possible, save the relevant part of decoded data into a safe buffer, In the case of a ring buffers, decoding buffer must be either :
and indicate where is its new address using LZ4_setStreamDecode() - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ4_setStreamDecode()
*/ */
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
@ -271,8 +280,8 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
Advanced decoding functions : Advanced decoding functions :
*_usingDict() : *_usingDict() :
These decoding functions work the same as These decoding functions work the same as
a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue() a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
They are stand-alone and don't use nor update an LZ4_streamDecode_t structure. They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
*/ */
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);

3
lib/lz4hc.c
View File

@ -620,7 +620,8 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
} }
/* Check if blocks follow each other */ /* Check if blocks follow each other */
if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); if ((const BYTE*)source != ctxPtr->end)
LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */ /* Check overlapping input/dictionary space */
{ {

130
programs/fuzzer.c
View File

@ -153,10 +153,10 @@ static U32 FUZ_rand(U32* src)
{ {
U32 rand32 = *src; U32 rand32 = *src;
rand32 *= PRIME1; rand32 *= PRIME1;
rand32 += PRIME2; rand32 ^= PRIME2;
rand32 = FUZ_rotl32(rand32, 13); rand32 = FUZ_rotl32(rand32, 13);
*src = rand32; *src = rand32;
return rand32 >> 3; return rand32;
} }
@ -168,27 +168,28 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
size_t pos = 0; size_t pos = 0;
U32 P32 = (U32)(32768 * proba); U32 P32 = (U32)(32768 * proba);
// First Byte /* First Bytes */
while (pos < 20)
BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize) while (pos < bufferSize)
{ {
// Select : Literal (noise) or copy (within 64K) /* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32) if (FUZ_RAND15BITS < P32)
{ {
// Copy (within 64K) /* Copy (within 64K) */
size_t match, d; size_t match, d;
size_t length = FUZ_RANDLENGTH + 4; size_t length = FUZ_RANDLENGTH + 4;
size_t offset = FUZ_RAND15BITS + 1; size_t offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos; while (offset > pos) offset >>= 1;
d = pos + length; d = pos + length;
if (d > bufferSize) d = bufferSize; while (d > bufferSize) d = bufferSize;
match = pos - offset; match = pos - offset;
while (pos < d) BBuffer[pos++] = BBuffer[match++]; while (pos < d) BBuffer[pos++] = BBuffer[match++];
} }
else else
{ {
// Literal (noise) /* Literal (noise) */
size_t d; size_t d;
size_t length = FUZ_RANDLENGTH; size_t length = FUZ_RANDLENGTH;
d = pos + length; d = pos + length;
@ -299,6 +300,16 @@ static void FUZ_displayUpdate(unsigned testNb)
} }
static void FUZ_findDiff(const void* buff1, const void* buff2)
{
const BYTE* b1 = (const BYTE*)buff1;
const BYTE* b2 = (const BYTE*)buff2;
size_t i=0;
while (b1[i]==b2[i]) i++;
DISPLAY("Wrong Byte at position %u\n", (unsigned)i);
}
static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration) static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration)
{ {
unsigned long long bytes = 0; unsigned long long bytes = 0;
@ -391,17 +402,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret; HCcompressedSize = ret;
// Test compression HC using external state /* Test compression HC using external state */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state /* Test compression using external state */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression /* Test compression */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize); ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
@ -411,7 +422,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
crcOrig = XXH32(block, blockSize, 0); crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work /* Test decoding with output size being exactly what's necessary => must work */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
@ -419,19 +430,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
crcCheck = XXH32(decodedBuffer, blockSize, 0); crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail /* Test decoding with one byte missing => must fail */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0; decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail /* Test decoding with one byte too much => must fail */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work /* Test decoding with output size exactly what's necessary => must work */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0; decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
@ -570,9 +581,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
/* Compress using External dictionary */ /* Compress using External dictionary */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
@ -593,7 +604,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external /* Decompress with dictionary as external */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0; decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
@ -601,11 +612,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0); crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig) if (crcCheck!=crcOrig)
{ FUZ_findDiff(block, decodedBuffer);
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
@ -619,7 +626,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0; decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
@ -640,9 +647,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
} }
// Compress HC using External dictionary /* Compress HC using External dictionary */
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 7); // even bigger separation dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
@ -667,14 +674,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0); crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig) if (crcCheck!=crcOrig)
{ FUZ_findDiff(block, decodedBuffer);
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
/* ***** End of tests *** */ /* ***** End of tests *** */
/* Fill stats */ /* Fill stats */
bytes += blockSize; bytes += blockSize;
@ -723,13 +725,13 @@ static void FUZ_unitTests(void)
char ringBuffer[ringBufferSize]; char ringBuffer[ringBufferSize];
U32 randState = 1; U32 randState = 1;
// Init /* Init */
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
// 32-bits address space overflow test /* 32-bits address space overflow test */
FUZ_AddressOverflow(); FUZ_AddressOverflow();
// LZ4 streaming tests /* LZ4 streaming tests */
{ {
LZ4_stream_t* statePtr; LZ4_stream_t* statePtr;
LZ4_stream_t streamingState; LZ4_stream_t streamingState;
@ -737,12 +739,12 @@ static void FUZ_unitTests(void)
U64 crcNew; U64 crcNew;
int result; int result;
// Allocation test /* Allocation test */
statePtr = LZ4_createStream(); statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr); LZ4_freeStream(statePtr);
// simple compression test /* simple compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0); crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState); LZ4_resetStream(&streamingState);
result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@ -753,7 +755,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0); crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// ring buffer test /* ring buffer test */
{ {
XXH64_state_t xxhOrig; XXH64_state_t xxhOrig;
XXH64_state_t xxhNew; XXH64_state_t xxhNew;
@ -787,7 +789,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64_digest(&xxhNew); crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// prepare next message /* prepare next message */
iNext += messageSize; iNext += messageSize;
rNext += messageSize; rNext += messageSize;
dNext += messageSize; dNext += messageSize;
@ -798,21 +800,20 @@ static void FUZ_unitTests(void)
} }
} }
// LZ4 HC streaming tests /* LZ4 HC streaming tests */
{ {
LZ4_streamHC_t* sp; LZ4_streamHC_t* sp;
LZ4_streamHC_t sHC; LZ4_streamHC_t sHC;
//XXH64_state_t xxh;
U64 crcOrig; U64 crcOrig;
U64 crcNew; U64 crcNew;
int result; int result;
// Allocation test /* Allocation test */
sp = LZ4_createStreamHC(); sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp); LZ4_freeStreamHC(sp);
// simple compression test /* simple HC compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0); crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0); LZ4_resetStreamHC(&sHC, 0);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@ -823,7 +824,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0); crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// simple dictionary compression test /* simple dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0); LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 64 KB); LZ4_loadDictHC(&sHC, testInput, 64 KB);
@ -855,7 +856,7 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
} }
// remote dictionary HC compression test /* remote dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0); LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB); LZ4_loadDictHC(&sHC, testInput, 32 KB);
@ -965,19 +966,46 @@ static void FUZ_unitTests(void)
XXH64_state_t xxhOrig; XXH64_state_t xxhOrig;
XXH64_state_t xxhNew; XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState; LZ4_streamDecode_t decodeState;
const U32 maxMessageSizeLog = 10; const U32 maxMessageSizeLog = 12;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1; const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
U32 totalMessageSize = 0; U32 totalMessageSize = 0;
U32 iNext = 0; U32 iNext = 0;
U32 dNext = 0; U32 dNext = 0;
const U32 dBufferSize = 64 KB + maxMessageSizeMask; const U32 dBufferSize = 64 KB;
XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0); XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0); LZ4_resetStreamHC(&sHC, 0);
LZ4_setStreamDecode(&decodeState, NULL, 0); LZ4_setStreamDecode(&decodeState, NULL, 0);
#define BSIZE1 65537
#define BSIZE2 16435
/* first block */
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
while (totalMessageSize < 9 MB) while (totalMessageSize < 9 MB)
{ {
XXH64_update(&xxhOrig, testInput + iNext, messageSize); XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@ -987,18 +1015,20 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize); XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew); crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); if (crcOrig != crcNew)
FUZ_findDiff(testInput + iNext, testVerify + dNext);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
/* prepare next message */ /* prepare next message */
dNext += messageSize; dNext += messageSize;
totalMessageSize += messageSize; totalMessageSize += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
iNext = (FUZ_rand(&randState) & 65535); iNext = (FUZ_rand(&randState) & 65535);
if (dNext + messageSize > dBufferSize) dNext = 0; if (dNext > dBufferSize) dNext = 0;
} }
} }
} }

32
programs/lz4io.c
View File

@ -443,7 +443,11 @@ static void LZ4IO_freeCResources(cRess_t ress)
if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
} }
/*
* LZ4IO_compressFilename_extRess()
* result : 0 : compression completed correctly
* 1 : missing or pb opening srcFileName
*/
static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel)
{ {
unsigned long long filesize = 0; unsigned long long filesize = 0;
@ -463,10 +467,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
memset(&prefs, 0, sizeof(prefs)); memset(&prefs, 0, sizeof(prefs));
/* File check */ /* File check */
{ if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1;
int srcFileAbsent = LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile);
if (srcFileAbsent) return srcFileAbsent;
}
/* Set compression parameters */ /* Set compression parameters */
prefs.autoFlush = 1; prefs.autoFlush = 1;
@ -547,11 +548,8 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* Final Status */ /* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
{
DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
(unsigned long long) filesize, (unsigned long long) compressedfilesize, filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */
(double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */
}
return 0; return 0;
} }
@ -588,8 +586,8 @@ int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel)
{ {
int i; int i;
int missing_files = 0; int missed_files = 0;
char* outFileName = (char*)malloc(FNSPACE); char* dstFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE; size_t ofnSize = FNSPACE;
const size_t suffixSize = strlen(suffix); const size_t suffixSize = strlen(suffix);
cRess_t ress; cRess_t ress;
@ -601,18 +599,18 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
for (i=0; i<ifntSize; i++) for (i=0; i<ifntSize; i++)
{ {
size_t ifnSize = strlen(inFileNamesTable[i]); size_t ifnSize = strlen(inFileNamesTable[i]);
if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); }
strcpy(outFileName, inFileNamesTable[i]); strcpy(dstFileName, inFileNamesTable[i]);
strcat(outFileName, suffix); strcat(dstFileName, suffix);
missing_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], outFileName, compressionLevel); missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
} }
/* Close & Free */ /* Close & Free */
LZ4IO_freeCResources(ress); LZ4IO_freeCResources(ress);
free(outFileName); free(dstFileName);
return missing_files; return missed_files;
} }