AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #33 from Cyan4973/streamHC

Browse Source 
Stream hc
This commit is contained in:
Yann Collet 2014-10-28 23:12:04 +01:00
commit eaccf78cfb
14 changed files with 1906 additions and 1028 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.travis.yml
View File

@ -1,14 +1,24 @@
language: c
compiler: gcc
script: make test
script: make test-travis
before_install:
- sudo apt-get update -qq
- sudo apt-get update -qq
- sudo apt-get install -qq gcc-multilib
- sudo apt-get install -qq valgrind
env:
- LZ4_TRAVIS_CI_ENV=-m32
- LZ4_TRAVIS_CI_ENV=-m64
- LZ4_TRAVIS_CI_ENV=-dist
- LZ4_TRAVIS_CI_ENV=-examples
- LZ4_TRAVIS_CI_ENV=-lz4
- LZ4_TRAVIS_CI_ENV=-lz4c
- LZ4_TRAVIS_CI_ENV=-lz4c32
- LZ4_TRAVIS_CI_ENV=-fullbench
- LZ4_TRAVIS_CI_ENV=-fullbench32
- LZ4_TRAVIS_CI_ENV=-fuzzer
- LZ4_TRAVIS_CI_ENV=-fuzzer32
- LZ4_TRAVIS_CI_ENV=-frametest
- LZ4_TRAVIS_CI_ENV=-frametest32
- LZ4_TRAVIS_CI_ENV=-mem
matrix:
fast_finish: true

20
Makefile
View File

@ -89,6 +89,16 @@ NONTEXT = images/image00.png images/image01.png images/image02.png \
SOURCES = $(TEXT) $(NONTEXT)
# Select test target for Travis CI's Build Matrix
ifeq ($(LZ4_TRAVIS_CI_ENV),-dist)
TRAVIS_TARGET=dist
else ifeq ($(LZ4_TRAVIS_CI_ENV),-examples)
TRAVIS_TARGET=examples
else
TRAVIS_TARGET=test-prg
endif
default: liblz4
@cd $(PRGDIR); $(MAKE) -e
@ -167,8 +177,14 @@ dist: clean
@echo Distribution $(DISTRIBNAME) built
test:
make dist
@cd examples; $(MAKE) -e $@
@cd $(PRGDIR); $(MAKE) -e $@
test-travis: $(TRAVIS_TARGET)
examples:
cd examples; $(MAKE) -e test
test-prg:
@cd $(PRGDIR); $(MAKE) -e test-travis
endif

244
examples/HCStreaming_ringBuffer.c Executable file
View File

@ -0,0 +1,244 @@
// LZ4 HC streaming API example : ring buffer
// Based on previous work from Takayuki Matsuoka
/**************************************
Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
# define snprintf sprintf_s
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
#endif
/**************************************
Includes
**************************************/
#include "lz4hc.h"
#include "lz4.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES
};
size_t write_int32(FILE* fp, int32_t i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
return fwrite(array, 1, arrayBytes, fp);
}
size_t read_int32(FILE* fp, int32_t* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, int arrayBytes) {
return fread(array, 1, arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_streamHC_t lz4Stream_body = { 0 };
LZ4_streamHC_t* lz4Stream = &lz4Stream_body;
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
unsigned done = 0;
for(;;)
{
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
{
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
done += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
inpOffset = 0;
}
}
write_int32(outFp, 0);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DEC_BUFFER_BYTES];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
unsigned done = 0;
for(;;)
{
int cmpBytes = 0;
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
size_t r1;
if(r0 != 1 || cmpBytes <= 0)
break;
r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes)
break;
}
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0)
break;
done += decBytes;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES)
decOffset = 0;
}
}
}
// Compare 2 files content
// return 0 if identical
// return ByteNb>0 if different
size_t compare(FILE* f0, FILE* f1)
{
size_t result = 1;
for (;;)
{
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
const size_t r1 = fread(b1, 1, sizeof(b1), f1);
if ((r0==0) && (r1==0)) return 0; // success
if (r0 != r1)
{
size_t smallest = r0;
if (r1<r0) smallest = r1;
result += smallest;
return result;
}
if (memcmp(b0, b1, r0))
{
unsigned errorPos = 0;
while ((errorPos < r0) && (b0[errorPos]==b1[errorPos])) errorPos++;
result += errorPos;
return result;
}
result += sizeof(b0);
}
return 1;
}
int main(int argc, char** argv)
{
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
unsigned fileID = 1;
unsigned pause = 0;
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
if (!strcmp(argv[1], "-p")) pause = 1, fileID = 2;
snprintf(inpFilename, 256, "%s", argv[fileID]);
snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[fileID], 9);
snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[fileID], 9);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
test_compress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
const size_t cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("Verify : OK\n");
} else {
printf("Verify : NG : error at pos %u\n", (unsigned)cmp-1);
}
fclose(decFp);
fclose(inpFp);
}
if (pause)
{
printf("Press enter to continue ...\n");
getchar();
}
return 0;
}

10
examples/Makefile
View File

@ -58,7 +58,7 @@ endif
default: all
all: printVersion doubleBuffer ringBuffer lineCompress
all: printVersion doubleBuffer ringBuffer ringBufferHC lineCompress
printVersion: $(LZ4DIR)/lz4.c printVersion.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@ -69,6 +69,9 @@ doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c
ringBuffer : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@ -76,10 +79,11 @@ test : all
./printVersion$(EXT)
./doubleBuffer$(EXT) $(TESTFILE)
./ringBuffer$(EXT) $(TESTFILE)
./ringBufferHC$(EXT) $(TESTFILE)
./lineCompress$(EXT) $(TESTFILE)
clean:
@rm -f core *.o *.dec *-0 *-8192 *.lz4s \
printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) lineCompress$(EXT)
@rm -f core *.o *.dec *-0 *-9 *-8192 *.lz4s \
printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT)
@echo Cleaning completed

155
lz4.c
View File

@ -876,8 +876,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
* in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int LZ4_decompress_generic(
const char* source,
char* dest,
const char* const source,
char* const dest,
int inputSize,
int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */
@ -885,20 +885,20 @@ FORCE_INLINE int LZ4_decompress_generic(
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const char* dictStart, /* only if dict==usingExtDict */
int dictSize /* note : = 0 if noDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize /* note : = 0 if noDict */
)
{
/* Local Variables */
const BYTE* restrict ip = (const BYTE*) source;
const BYTE* ref;
const BYTE* const iend = ip + inputSize;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + outputSize;
BYTE* cpy;
BYTE* oexit = op + targetOutputSize;
const BYTE* const lowLimit = (const BYTE*)dest - dictSize;
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4};
@ -919,8 +919,9 @@ FORCE_INLINE int LZ4_decompress_generic(
{
unsigned token;
size_t length;
const BYTE* match;
/* get runlength */
/* get literal length */
token = *ip++;
if ((length=(token>>ML_BITS)) == RUN_MASK)
{
@ -931,8 +932,8 @@ FORCE_INLINE int LZ4_decompress_generic(
length += s;
}
while (likely((endOnInput)?ip<iend-RUN_MASK:1) && (s==255));
if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
}
/* copy literals */
@ -958,8 +959,8 @@ FORCE_INLINE int LZ4_decompress_generic(
LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;
/* get offset */
LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
if ((checkOffset) && (unlikely(ref < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */
LZ4_READ_LITTLEENDIAN_16(match,cpy,ip); ip+=2;
if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */
/* get matchlength */
if ((length=(token&ML_MASK)) == ML_MASK)
@ -971,36 +972,38 @@ FORCE_INLINE int LZ4_decompress_generic(
s = *ip++;
length += s;
} while (s==255);
if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;
/* check external dictionary */
if ((dict==usingExtDict) && (ref < (BYTE* const)dest))
if ((dict==usingExtDict) && (match < lowPrefix))
{
if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error;
if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */
if (length <= (size_t)(dest-(char*)ref))
if (length <= (size_t)(lowPrefix-match))
{
ref = dictEnd - (dest-(char*)ref);
memcpy(op, ref, length);
/* match can be copied as a single segment from external dictionary */
match = dictEnd - (lowPrefix-match);
memcpy(op, match, length);
op += length;
}
else
{
size_t copySize = (size_t)(dest-(char*)ref);
/* match encompass external dictionary and current segment */
size_t copySize = (size_t)(lowPrefix-match);
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
copySize = length - copySize;
if (copySize > (size_t)((char*)op-dest)) /* overlap */
if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */
{
BYTE* const endOfMatch = op + copySize;
const BYTE* copyFrom = (BYTE*)dest;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
}
else
{
memcpy(op, dest, copySize);
memcpy(op, lowPrefix, copySize);
op += copySize;
}
}
@ -1009,25 +1012,25 @@ FORCE_INLINE int LZ4_decompress_generic(
/* copy repeated sequence */
cpy = op + length;
if (unlikely((op-ref)<(int)STEPSIZE))
if (unlikely((op-match)<(int)STEPSIZE))
{
const size_t dec64 = dec64table[op-ref];
op[0] = ref[0];
op[1] = ref[1];
op[2] = ref[2];
op[3] = ref[3];
ref += dec32table[op-ref];
A32(op+4) = A32(ref);
op += 8; ref -= dec64;
} else { LZ4_COPY8(op,ref); }
const size_t dec64 = dec64table[op-match];
op[0] = match[0];
op[1] = match[1];
op[2] = match[2];
op[3] = match[3];
match += dec32table[op-match];
A32(op+4) = A32(match);
op += 8; match -= dec64;
} else { LZ4_COPY8(op,match); }
if (unlikely(cpy>oend-12))
{
if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */
if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, ref, (oend-COPYLENGTH));
while(op<cpy) *op++=*ref++;
if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, match, (oend-COPYLENGTH));
while(op<cpy) *op++=*match++;
}
else LZ4_WILDCOPY(op, ref, cpy);
else LZ4_WILDCOPY(op, match, cpy);
op=cpy; /* correction */
}
@ -1045,25 +1048,28 @@ _output_error:
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, NULL, 0);
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
}
int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, NULL, 0);
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
}
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
}
/* streaming decompression functions */
typedef struct
{
const char* dictionary;
int dictSize;
BYTE* externalDict;
size_t extDictSize;
BYTE* prefixEnd;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
/*
@ -1074,7 +1080,6 @@ typedef struct
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U32), LZ4_STREAMDECODESIZE_U32);
MEM_INIT(lz4s, 0, LZ4_STREAMDECODESIZE);
return lz4s;
}
@ -1094,8 +1099,10 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
{
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
lz4sd->dictionary = dictionary;
lz4sd->dictSize = dictSize;
lz4sd->prefixSize = (size_t) dictSize;
lz4sd->prefixEnd = (BYTE*) dictionary + dictSize;
lz4sd->externalDict = NULL;
lz4sd->extDictSize = 0;
return 1;
}
@ -1104,23 +1111,32 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks must still be available at the memory position where they were decoded.
If it's not possible, save the relevant part of decoded data into a safe buffer,
and indicate where it stands using LZ4_setDictDecode()
and indicate where it stands using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
int result;
result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
if (result <= 0) return result;
if (lz4sd->dictionary + lz4sd->dictSize == dest)
if (lz4sd->prefixEnd == (BYTE*)dest)
{
lz4sd->dictSize += result;
result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
endOnInputSize, full, 0,
usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize += result;
lz4sd->prefixEnd += result;
}
else
{
lz4sd->dictionary = dest;
lz4sd->dictSize = result;
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
endOnInputSize, full, 0,
usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize = result;
lz4sd->prefixEnd = (BYTE*)dest + result;
}
return result;
@ -1131,16 +1147,25 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
int result;
result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
if (result <= 0) return result;
if (lz4sd->dictionary + lz4sd->dictSize == dest)
if (lz4sd->prefixEnd == (BYTE*)dest)
{
lz4sd->dictSize += result;
result = LZ4_decompress_generic(source, dest, 0, originalSize,
endOnOutputSize, full, 0,
usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize += originalSize;
lz4sd->prefixEnd += originalSize;
}
else
{
lz4sd->dictionary = dest;
lz4sd->dictSize = result;
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize;
result = LZ4_decompress_generic(source, dest, 0, originalSize,
endOnOutputSize, full, 0,
usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize = originalSize;
lz4sd->prefixEnd = (BYTE*)dest + originalSize;
}
return result;
@ -1157,28 +1182,30 @@ Advanced decoding functions :
FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
{
if (dictSize==0)
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, NULL, 64 KB);
if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1)))
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, NULL, 64 KB);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, dictStart, dictSize);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
if (dictStart+dictSize == dest)
{
if (dictSize >= (int)(64 KB - 1))
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
}
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize);
}
int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
//return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
}
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
//return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize);
return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
}
/* debug function */
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize);
}
@ -1258,10 +1285,10 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char*
int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
}
int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
}

38
lz4.h
View File

@ -193,7 +193,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr);
* LZ4_freeStream releases its memory.
*/
LZ4_stream_t* LZ4_createStream(void);
int LZ4_freeStream (LZ4_stream_t* LZ4_stream);
int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr);
/*
* LZ4_loadDict
@ -202,21 +202,21 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream);
* Loading a size of 0 is allowed.
* Return : 1 if OK, 0 if error
*/
int LZ4_loadDict (LZ4_stream_t* LZ4_stream, const char* dictionary, int dictSize);
int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize);
/*
* LZ4_compress_continue
* Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
* Previous data blocks are assumed to still be present at their previous location.
*/
int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
/*
* LZ4_compress_limitedOutput_continue
* Same as before, but also specify a maximum target compressed size (maxOutputSize)
* If objective cannot be met, compression exits, and returns a zero.
*/
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/*
* LZ4_saveDict
@ -227,14 +227,14 @@ int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* s
* Return : dictionary size in bytes, or 0 if error
* Note : any dictSize > 64 KB will be interpreted as 64KB.
*/
int LZ4_saveDict (LZ4_stream_t* LZ4_stream, char* safeBuffer, int dictSize);
int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize);
/************************************************
Experimental Streaming Decompression Functions
************************************************/
#define LZ4_STREAMDECODESIZE_U32 4
#define LZ4_STREAMDECODESIZE_U32 8
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
/*
* LZ4_streamDecode_t
@ -243,16 +243,6 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_stream, char* safeBuffer, int dictSize);
*/
typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
/*
* LZ4_setStreamDecode
* Use this function to instruct where to find the dictionary.
* This function can be used to specify a static dictionary,
* or to instruct where to find some previously decoded data saved into a different memory space.
* Setting a size of 0 is allowed (same effect as no dictionary).
* Return : 1 if OK, 0 if error
*/
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
/*
* If you prefer dynamic allocation methods,
* LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
@ -261,12 +251,22 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
LZ4_streamDecode_t* LZ4_createStreamDecode(void);
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
/*
* LZ4_setStreamDecode
* Use this function to instruct where to find the dictionary.
* This function can be used to specify a static dictionary,
* or to instruct where to find some previously decoded data saved into a different memory space.
* Setting a size of 0 is allowed (same effect as no dictionary, same effect as reset).
* Return : 1 if OK, 0 if error
*/
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks must still be available at the memory position where they were decoded.
If it's not possible, save the relevant part of decoded data into a safe buffer,
and indicate where its new address using LZ4_setStreamDecode()
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
If this condition is not possible, save the relevant part of decoded data into a safe buffer,
and indicate where is its new address using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);

905
lz4frame.c
View File

File diff suppressed because it is too large Load Diff

2
lz4frame.h
View File

@ -70,7 +70,7 @@ typedef size_t LZ4F_errorCode_t;
ITEM(ERROR_maxCode)
#define LZ4F_GENERATE_ENUM(ENUM) ENUM,
typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to let programmer detect & handle specific errors */
typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */
const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return enum as string */

683
lz4hc.c
View File

@ -1,46 +1,46 @@
/*
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
/**************************************
Tuning Parameter
Tuning Parameter
**************************************/
#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8
/**************************************
Memory routines
Memory routines
**************************************/
#include <stdlib.h> /* calloc, free */
#define ALLOCATOR(s) calloc(1,s)
@ -50,25 +50,25 @@
/**************************************
CPU Feature Detection
CPU Feature Detection
**************************************/
/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|| defined(__64BIT__) || defined(__mips64) \
|| defined(__powerpc64__) || defined(__powerpc64le__) \
|| defined(__ppc64__) || defined(__ppc64le__) \
|| defined(__PPC64__) || defined(__PPC64LE__) \
|| defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \
|| defined(__s390x__) ) /* Detects 64 bits mode */
|| defined(__64BIT__) || defined(__mips64) \
|| defined(__powerpc64__) || defined(__powerpc64le__) \
|| defined(__ppc64__) || defined(__ppc64le__) \
|| defined(__PPC64__) || defined(__PPC64LE__) \
|| defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \
|| defined(__s390x__) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
# define LZ4_ARCH64 0
#endif
/*
* Little Endian or Big Endian ?
* Overwrite the #define below if you know your architecture endianess
*/
* Little Endian or Big Endian ?
* Overwrite the #define below if you know your architecture endianess
*/
#include <stdlib.h> /* Apparently required to detect endianess */
#if defined (__GLIBC__)
# include <endian.h>
@ -78,19 +78,19 @@
#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))
# define LZ4_BIG_ENDIAN 1
#elif defined(__sparc) || defined(__sparc__) \
|| defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \
|| defined(__hpux) || defined(__hppa) \
|| defined(_MIPSEB) || defined(__s390__)
|| defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \
|| defined(__hpux) || defined(__hppa) \
|| defined(_MIPSEB) || defined(__s390__)
# define LZ4_BIG_ENDIAN 1
#else
/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */
#endif
/*
* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
*/
* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
*/
#if defined(__ARM_FEATURE_UNALIGNED)
# define LZ4_FORCE_UNALIGNED_ACCESS 1
#endif
@ -102,7 +102,7 @@
/**************************************
Compiler Options
Compiler Options
**************************************/
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
/* "restrict" is a known keyword */
@ -138,28 +138,28 @@
/**************************************
Includes
Includes
**************************************/
#include "lz4hc.h"
#include "lz4.h"
/**************************************
Basic Types
Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)
@ -190,7 +190,7 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
/**************************************
Constants
Constants
**************************************/
#define MINMATCH 4
@ -214,28 +214,24 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define MINLENGTH (MFLIMIT+1)
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
#define KB *(1U<<10)
#define MB *(1U<<20)
#define KB *(1<<10)
#define MB *(1<<20)
#define GB *(1U<<30)
/**************************************
Architecture-specific macros
Architecture-specific macros
**************************************/
#if LZ4_ARCH64 /* 64-bit */
# define STEPSIZE 8
# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
# define AARCH A64
# define HTYPE U32
# define INITBASE(b,s) const BYTE* const b = s
# define AARCH A64
#else /* 32-bit */
# define STEPSIZE 4
# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
# define AARCH A32
# define HTYPE U32
# define INITBASE(b,s) const BYTE* const b = s
# define AARCH A32
#endif
#if defined(LZ4_BIG_ENDIAN)
@ -248,33 +244,37 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
/**************************************
Local Types
Local Types
**************************************/
typedef struct
{
U32 hashTable[HASHTABLESIZE];
U16 chainTable[MAXD];
const BYTE* inputBuffer;
const BYTE* base;
const BYTE* end;
HTYPE hashTable[HASHTABLESIZE];
U16 chainTable[MAXD];
const BYTE* nextToUpdate;
const BYTE* dictBase;
U32 dictLimit;
U32 nextToUpdate;
U32 compressionLevel;
U32 lowLimit;
} LZ4HC_Data_Structure;
/**************************************
Macros
Macros
**************************************/
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* Visual : use only *after* variable declarations */
#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
#define HASH_VALUE(p) HASH_FUNCTION(A32(p))
#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base)
#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(A32(ptr)); }
/**************************************
Private functions
Private functions
**************************************/
#if LZ4_ARCH64
@ -342,225 +342,197 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
#endif
int LZ4_sizeofStreamStateHC()
{
return sizeof(LZ4HC_Data_Structure);
}
FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
FORCE_INLINE void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
hc4->nextToUpdate = base + 1;
hc4->base = base;
hc4->nextToUpdate = 64 KB;
hc4->base = base - 64 KB;
hc4->inputBuffer = base;
hc4->end = base;
}
int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
{
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
return 0;
}
void* LZ4_createHC (const char* inputBuffer)
{
void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
return hc4;
}
int LZ4_freeHC (void* LZ4HC_Data)
{
FREEMEM(LZ4HC_Data);
return (0);
hc4->dictBase = base - 64 KB;
hc4->dictLimit = 64 KB;
hc4->lowLimit = 64 KB;
}
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
{
U16* chainTable = hc4->chainTable;
HTYPE* HashTable = hc4->hashTable;
INITBASE(base,hc4->base);
U16* chainTable = hc4->chainTable;
U32* HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const U32 target = (U32)(ip - base);
U32 idx = hc4->nextToUpdate;
while(hc4->nextToUpdate < ip)
while(idx < target)
{
const BYTE* const p = hc4->nextToUpdate;
size_t delta = (p) - HASH_POINTER(p);
U32 h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - HashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
DELTANEXT(p) = (U16)delta;
HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base);
hc4->nextToUpdate++;
chainTable[idx & 0xFFFF] = (U16)delta;
HashTable[h] = idx;
idx++;
}
hc4->nextToUpdate = target;
}
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
{
LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer;
if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */
distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */
LZ4HC_Insert(hc4, hc4->end - MINMATCH);
memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
hc4->nextToUpdate -= distance;
hc4->base -= distance;
if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */
{
int i;
hc4->base += 1 GB;
for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;
}
hc4->end -= distance;
return (char*)(hc4->end);
if (ctxPtr->end >= ctxPtr->base + 4)
LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content
// Note : need to handle risk of index overflow
// Use only one memory segment for dict, so any previous External Dict is lost at this stage
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = newBlock - ctxPtr->dictLimit;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block
}
FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit)
{
const BYTE* p1t = p1;
const BYTE* const p1Start = p1;
while (p1t<matchlimit-(STEPSIZE-1))
while (p1 <= p1Limit - STEPSIZE)
{
size_t diff = AARCH(p2) ^ AARCH(p1t);
if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; }
p1t += LZ4_NbCommonBytes(diff);
return (p1t - p1);
size_t diff = AARCH(p2) ^ AARCH(p1);
if (!diff) { p1+=STEPSIZE; p2+=STEPSIZE; continue; }
p1 += LZ4_NbCommonBytes(diff);
return (p1 - p1Start);
}
if (LZ4_ARCH64) if ((p1t<(matchlimit-3)) && (A32(p2) == A32(p1t))) { p1t+=4; p2+=4; }
if ((p1t<(matchlimit-1)) && (A16(p2) == A16(p1t))) { p1t+=2; p2+=2; }
if ((p1t<matchlimit) && (*p2 == *p1t)) p1t++;
return (p1t - p1);
if (LZ4_ARCH64) if ((p1<(p1Limit-3)) && (A32(p2) == A32(p1))) { p1+=4; p2+=4; }
if ((p1<(p1Limit-1)) && (A16(p2) == A16(p1))) { p1+=2; p2+=2; }
if ((p1<p1Limit) && (*p2 == *p1)) p1++;
return (p1 - p1Start);
}
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos, const int maxNbAttempts)
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
HTYPE* const HashTable = hc4->hashTable;
const BYTE* ref;
INITBASE(base,hc4->base);
U32* const HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const BYTE* const dictBase = hc4->dictBase;
const U32 dictLimit = hc4->dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
U32 matchIndex;
const BYTE* match;
int nbAttempts=maxNbAttempts;
size_t repl=0, ml=0;
U16 delta=0; /* useless assignment, to remove an uninitialization warning */
size_t ml=0;
/* HC4 match finder */
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
#define REPEAT_OPTIMIZATION
#ifdef REPEAT_OPTIMIZATION
/* Detect repetitive sequences of length <= 4 */
if ((U32)(ip-ref) <= 4) /* potential repetition */
{
if (A32(ref) == A32(ip)) /* confirmed */
{
delta = (U16)(ip-ref);
repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
*matchpos = ref;
}
ref = GETNEXT(ref);
}
#endif
while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
while ((matchIndex>=lowLimit) && (nbAttempts))
{
nbAttempts--;
if (*(ref+ml) == *(ip+ml))
if (A32(ref) == A32(ip))
if (matchIndex >= dictLimit)
{
size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = ref; }
match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (A32(match) == A32(ip)))
{
size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
}
ref = GETNEXT(ref);
}
#ifdef REPEAT_OPTIMIZATION
/* Complete table */
if (repl)
{
const BYTE* ptr = ip;
const BYTE* end;
end = ip + repl - (MINMATCH-1);
while(ptr < end-delta)
else
{
DELTANEXT(ptr) = delta; /* Pre-Load */
ptr++;
match = dictBase + matchIndex;
if (A32(match) == A32(ip))
{
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iLimit))
mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iLimit);
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos
}
}
do
{
DELTANEXT(ptr) = delta;
HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */
ptr++;
} while(ptr < end);
hc4->nextToUpdate = end;
matchIndex -= chainTable[matchIndex & 0xFFFF];
}
#endif
return (int)ml;
}
FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts)
FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
LZ4HC_Data_Structure* hc4,
const BYTE* ip,
const BYTE* iLowLimit,
const BYTE* iHighLimit,
int longest,
const BYTE** matchpos,
const BYTE** startpos,
const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
HTYPE* const HashTable = hc4->hashTable;
INITBASE(base,hc4->base);
const BYTE* ref;
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const U32 dictLimit = hc4->dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
const BYTE* const dictBase = hc4->dictBase;
const BYTE* match;
U32 matchIndex;
int nbAttempts = maxNbAttempts;
int delta = (int)(ip-startLimit);
int delta = (int)(ip-iLowLimit);
/* First Match */
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
while ((matchIndex>=lowLimit) && (nbAttempts))
{
nbAttempts--;
if (*(startLimit + longest) == *(ref - delta + longest))
if (A32(ref) == A32(ip))
if (matchIndex >= dictLimit)
{
#if 1
const BYTE* reft = ref+MINMATCH;
const BYTE* ipt = ip+MINMATCH;
const BYTE* startt = ip;
match = base + matchIndex;
if (*(iLowLimit + longest) == *(match - delta + longest))
if (A32(match) == A32(ip))
{
const BYTE* startt = ip;
const BYTE* tmpMatch = match;
const BYTE* const matchEnd = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iHighLimit);
while (ipt<matchlimit-(STEPSIZE-1))
while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;}
if ((matchEnd-startt) > longest)
{
longest = (int)(matchEnd-startt);
*matchpos = tmpMatch;
*startpos = startt;
}
}
}
else
{
match = dictBase + matchIndex;
if (A32(match) == A32(ip))
{
size_t diff = AARCH(reft) ^ AARCH(ipt);
if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }
ipt += LZ4_NbCommonBytes(diff);
goto _endCount;
}
if (LZ4_ARCH64) if ((ipt<(matchlimit-3)) && (A32(reft) == A32(ipt))) { ipt+=4; reft+=4; }
if ((ipt<(matchlimit-1)) && (A16(reft) == A16(ipt))) { ipt+=2; reft+=2; }
if ((ipt<matchlimit) && (*reft == *ipt)) ipt++;
_endCount:
reft = ref;
#else
/* Easier for code maintenance, but unfortunately slower too */
const BYTE* startt = ip;
const BYTE* reft = ref;
const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);
#endif
while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;}
if ((ipt-startt) > longest)
{
longest = (int)(ipt-startt);
*matchpos = reft;
*startpos = startt;
size_t mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
ref = GETNEXT(ref);
matchIndex -= chainTable[matchIndex & 0xFFFF];
}
return longest;
@ -569,22 +541,26 @@ _endCount:
typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
//static unsigned debug = 0;
FORCE_INLINE int LZ4HC_encodeSequence (
const BYTE** ip,
BYTE** op,
const BYTE** anchor,
int matchLength,
const BYTE* ref,
limitedOutput_directive limitedOutputBuffer,
BYTE* oend)
const BYTE** ip,
BYTE** op,
const BYTE** anchor,
int matchLength,
const BYTE* const match,
limitedOutput_directive limitedOutputBuffer,
BYTE* oend)
{
int length;
BYTE* token;
//if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); // debug
/* Encode Literal length */
length = (int)(*ip - *anchor);
token = (*op)++;
if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */
if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
else *token = (BYTE)(length<<ML_BITS);
@ -592,11 +568,11 @@ FORCE_INLINE int LZ4HC_encodeSequence (
LZ4_BLINDCOPY(*anchor, *op, length);
/* Encode Offset */
LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-match));
/* Encode MatchLength */
length = (int)(matchLength-MINMATCH);
if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; /* Check output limit */
if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
else *token += (BYTE)(length);
@ -610,14 +586,14 @@ FORCE_INLINE int LZ4HC_encodeSequence (
#define MAX_COMPRESSION_LEVEL 16
static int LZ4HC_compress_generic (
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
int compressionLevel,
limitedOutput_directive limit
)
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
int compressionLevel,
limitedOutput_directive limit
)
{
LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
const BYTE* ip = (const BYTE*) source;
@ -629,7 +605,7 @@ static int LZ4HC_compress_generic (
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<<LZ4HC_DEFAULT_COMPRESSIONLEVEL;
unsigned maxNbAttempts;
int ml, ml2, ml3, ml0;
const BYTE* ref=NULL;
const BYTE* start2=NULL;
@ -640,8 +616,10 @@ static int LZ4HC_compress_generic (
const BYTE* ref0;
/* Ensure blocks follow each other */
if (ip != ctx->end) return 0;
/* init */
if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL;
if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL;
maxNbAttempts = 1 << compressionLevel;
ctx->end += inputSize;
ip++;
@ -689,10 +667,10 @@ _Search2:
_Search3:
/*
* Currently we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
* Currently we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
if ((start2 - ip) < OPTIMAL_ML)
{
int correction;
@ -760,9 +738,9 @@ _Search3:
}
/*
* OK, now we have 3 ascending matches; let's write at least the first one
* ip & ref are known; Now for ml
*/
* OK, now we have 3 ascending matches; let's write at least the first one
* ip & ref are known; Now for ml
*/
if (start2 < ip+ml)
{
if ((start2 - ip) < (int)ML_MASK)
@ -794,7 +772,6 @@ _Search3:
ml2 = ml3;
goto _Search3;
}
/* Encode Last Literals */
@ -814,28 +791,18 @@ _Search3:
int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel)
{
void* ctx = LZ4_createHC(source);
int result;
if (ctx==NULL) return 0;
result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
LZ4_freeHC(ctx);
return result;
LZ4HC_Data_Structure ctx;
LZ4HC_init(&ctx, (const BYTE*)source);
return LZ4HC_compress_generic (&ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); }
int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
void* ctx = LZ4_createHC(source);
int result;
if (ctx==NULL) return 0;
result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
LZ4_freeHC(ctx);
return result;
LZ4HC_Data_Structure ctx;
LZ4HC_init(&ctx, (const BYTE*)source);
return LZ4HC_compress_generic (&ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
@ -845,15 +812,15 @@ int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize,
/*****************************
Using external allocation
Using external allocation
*****************************/
int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); }
int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); }
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit);
}
@ -864,7 +831,7 @@ int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int
int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
@ -872,26 +839,158 @@ int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, c
{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); }
/****************************
Stream functions
****************************/
/**************************************
Experimental Streaming Functions
**************************************/
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; };
/* initialization */
void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= LZ4_STREAMHCSIZE); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL;
((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel;
}
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) dictionary);
if (dictSize >= 4) LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3));
((LZ4HC_Data_Structure*) LZ4_streamHCPtr)->end = (const BYTE*)dictionary + dictSize;
return 1;
}
/* compression */
static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr,
const char* source, char* dest,
int inputSize, int maxOutputSize, limitedOutput_directive limit)
{
/* auto-init if forgotten */
if (dsPtr->base == NULL)
LZ4HC_init (dsPtr, (const BYTE*) source);
/* Check overflow */
if ((size_t)(dsPtr->end - dsPtr->base) > 2 GB)
{
size_t dictSize = (size_t)(dsPtr->end - dsPtr->base) - dsPtr->dictLimit;
if (dictSize > 64 KB) dictSize = 64 KB;
LZ4_loadDictHC((LZ4_streamHC_t*)dsPtr, (const char*)(dsPtr->end) - dictSize, (int)dictSize);
}
/* Check if blocks follow each other */
if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */
{
const BYTE* sourceEnd = (const BYTE*) source + inputSize;
const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit;
const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd))
{
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase);
if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit;
}
}
return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit);
}
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
{
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
}
/* dictionary saving */
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_Data_Structure* sp = (LZ4HC_Data_Structure*)LZ4_streamHCPtr;
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 0) dictSize = 0;
if (dictSize > (sp->end - (sp->base + sp->lowLimit))) dictSize = (int)(sp->end - (sp->base + sp->lowLimit));
memcpy(safeBuffer, sp->end - dictSize, dictSize);
LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize);
return dictSize;
}
/***********************************
Deprecated Streaming functions
***********************************/
int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
{
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
return 0;
}
void* LZ4_createHC (const char* inputBuffer)
{
void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
return hc4;
}
int LZ4_freeHC (void* LZ4HC_Data)
{
FREEMEM(LZ4HC_Data);
return (0);
}
/*
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
}
*/
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer;
if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */
distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */
LZ4HC_Insert(hc4, hc4->end - MINMATCH);
memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
hc4->base -= distance;
if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */
{
int i;
hc4->base += 1 GB;
for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;
}
hc4->end -= distance;
return (char*)(hc4->end);
}

62
lz4hc.h
View File

@ -74,7 +74,7 @@ int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize
*/
/* Note :
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
*/
@ -95,19 +95,69 @@ int LZ4_sizeofStateHC();
Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
The allocated memory can be provided to the compressions functions using 'void* state' parameter.
The allocated memory can be provided to the compression functions using 'void* state' parameter.
LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap).
*/
/**************************************
Streaming Functions
Experimental Streaming Functions
**************************************/
#define LZ4_STREAMHCSIZE_U32 65548
#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U32 * sizeof(unsigned int))
typedef struct { unsigned int table[LZ4_STREAMHCSIZE_U32]; } LZ4_streamHC_t;
/*
This structure allows static allocation of LZ4 HC streaming state.
State must then be initialized using LZ4_resetStreamHC() before first use.
If you prefer dynamic allocation, please refer to functions below.
*/
LZ4_streamHC_t* LZ4_createStreamHC(void);
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr);
/*
These functions create and release memory for LZ4 HC streaming state.
Newly created states are already initialized.
Existing state space can be re-used anytime using LZ4_resetStreamHC().
*/
void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize);
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize);
/*
These functions compress data in successive blocks of any size, using previous blocks as dictionary.
One key assumption is that each previous block will remain read-accessible while compressing next block.
Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block.
They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
If, for any reason, previous data block can't be preserved in memory during next compression block,
you can save it to a safer memory space,
using LZ4_saveDictHC().
*/
/**************************************
Deprecated Streaming Functions
**************************************/
/* Note : these streaming functions still follows the older model */
void* LZ4_createHC (const char* inputBuffer);
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
//int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
//int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
int LZ4_freeHC (void* LZ4HC_Data);

70
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
RELEASE=r123
RELEASE=r124
DESTDIR?=
PREFIX ?= /usr
@ -45,15 +45,6 @@ LZ4DIR=..
TEST_FILES = COPYING
TEST_TARGETS=test-native
BENCH_NB=-i5
# Minimize test target for Travis CI's Build Matrix
ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
TEST_TARGETS=test-force32
BENCH_NB=-i1
else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
BENCH_NB=-i1
endif
# Define *.exe as extension for Windows systems
@ -66,9 +57,35 @@ VOID = /dev/null
endif
# Select test target for Travis CI's Build Matrix
ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4)
TRAVIS_TARGET=test-lz4
else ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4c)
TRAVIS_TARGET=test-lz4c
else ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4c32)
TRAVIS_TARGET=test-lz4c32
else ifeq ($(LZ4_TRAVIS_CI_ENV),-fullbench)
TRAVIS_TARGET=test-fullbench
else ifeq ($(LZ4_TRAVIS_CI_ENV),-fullbench32)
TRAVIS_TARGET=test-fullbench32
else ifeq ($(LZ4_TRAVIS_CI_ENV),-fuzzer)
TRAVIS_TARGET=test-fuzzer
else ifeq ($(LZ4_TRAVIS_CI_ENV),-fuzzer32)
TRAVIS_TARGET=test-fuzzer32
else ifeq ($(LZ4_TRAVIS_CI_ENV),-frametest)
TRAVIS_TARGET=test-frametest
else ifeq ($(LZ4_TRAVIS_CI_ENV),-frametest32)
TRAVIS_TARGET=test-frametest32
else ifeq ($(LZ4_TRAVIS_CI_ENV),-mem)
TRAVIS_TARGET=test-mem
else
TRAVIS_TARGET=test
endif
default: lz4 lz4c
all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest datagen
all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest frametest32 datagen
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c
$(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
@ -94,6 +111,9 @@ fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
datagen : datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@ -103,7 +123,8 @@ clean:
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) \
frametest$(EXT) datagen$(EXT)
frametest$(EXT) frametest32$(EXT) \
datagen$(EXT)
@echo Cleaning completed
@ -132,13 +153,13 @@ uninstall:
[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
@echo lz4 successfully uninstalled
test-native: test-lz4 test-lz4c test-frame test-fullbench test-fuzzer test-mem
test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem
test-force32: test-lz4c32 test-fullbench32 test-fuzzer32 test-mem32
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32
test-all: test-native test-force32
test-all: test test32
test: $(TEST_TARGETS)
test-travis: $(TRAVIS_TARGET)
test-lz4: lz4 datagen
./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID)
@ -158,10 +179,10 @@ test-lz4c32: lz4 lz4c32 lz4 datagen
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
test-fullbench: fullbench
./fullbench --no-prompt $(BENCH_NB) $(TEST_FILES)
./fullbench --no-prompt $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES)
./fullbench32 --no-prompt $(TEST_FILES)
test-fuzzer: fuzzer
./fuzzer
@ -169,18 +190,21 @@ test-fuzzer: fuzzer
test-fuzzer32: fuzzer32
./fuzzer32
test-frame: frametest
test-frametest: frametest
./frametest
test-frametest32: frametest32
./frametest32
test-mem: lz4 datagen frametest
./datagen -g16KB > tmp
valgrind ./lz4 -9 -BD -f tmp /dev/null
valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null
./datagen -g16MB > tmp
valgrind ./lz4 -9 -B5D -f tmp /dev/null
valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null
./datagen -g256MB > tmp
valgrind ./lz4 -B4D -f tmp /dev/null
valgrind --leak-check=yes ./lz4 -B4D -f tmp /dev/null
rm tmp
valgrind ./frametest -i100
valgrind --leak-check=yes ./frametest -i100
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...

26
programs/frametest.c
View File

@ -78,7 +78,7 @@ typedef unsigned long long U64;
#define MB *(1U<<20)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 128 KB;
static const U32 nbTestsDefault = 256 KB;
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
@ -426,10 +426,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
LZ4F_preferences_t* prefsPtr = &prefs;
(void)FUZ_rand(&coreRand); // update rand seed
prefs.frameInfo.blockMode = BMId;
prefs.frameInfo.blockSizeID = BSId;
prefs.frameInfo.contentChecksumFlag = CCflag;
prefs.frameInfo.blockMode = (blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
prefs.autoFlush = autoflush;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
if ((FUZ_rand(&randState)&0xF) == 1) prefsPtr = NULL;
DISPLAYUPDATE(2, "\r%5u ", testNb);
@ -444,7 +445,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
{
const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
const BYTE* const iend = ip + srcSize;
BYTE* op = compressedBuffer;
BYTE* op = (BYTE*)compressedBuffer;
BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
unsigned maxBits = FUZ_highbit((U32)srcSize);
result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
@ -478,9 +479,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
}
{
const BYTE* ip = compressedBuffer;
const BYTE* ip = (const BYTE*)compressedBuffer;
const BYTE* const iend = ip + cSize;
BYTE* op = decodedBuffer;
BYTE* op = (BYTE*)decodedBuffer;
BYTE* const oend = op + srcDataLength;
unsigned maxBits = FUZ_highbit((U32)cSize);
unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
@ -597,6 +598,11 @@ int main(int argc, char** argv)
argument++;
displayLevel--;
break;
case 'p': /* pause at the end */
argument++;
pause = 1;
break;
case 'i':
argument++;
nbTests=0;
@ -628,7 +634,7 @@ int main(int argc, char** argv)
argument++;
}
break;
case 'p': /* compressibility % */
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
@ -640,10 +646,6 @@ int main(int argc, char** argv)
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
case 'P': /* pause at the end */
argument++;
pause = 1;
break;
default:
;
return FUZ_usage();

4
programs/fullbench.c
View File

@ -313,12 +313,12 @@ static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inS
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_continue(ctx, in, out, inSize);
return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize);
}
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)

697
programs/fuzzer.c
View File

@ -1,29 +1,29 @@
/*
fuzzer.c - Fuzzer test tool for LZ4
Copyright (C) Yann Collet 2012-2014
GPL v2 License
fuzzer.c - Fuzzer test tool for LZ4
Copyright (C) Yann Collet 2012-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
/**************************************
Remove Visual warning messages
Remove Visual warning messages
**************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets
#ifdef _MSC_VER /* Visual Studio */
@ -34,7 +34,7 @@
/**************************************
Includes
Includes
**************************************/
#include <stdlib.h>
#include <stdio.h> // fgets, sscanf
@ -46,26 +46,26 @@
/**************************************
Basic Types
Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
Constants
Constants
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION ""
@ -87,21 +87,15 @@
/*****************************************
Macros
Macros
*****************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static const U32 refreshRate = 250;
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static int g_displayLevel = 2;
static const U32 g_refreshRate = 250;
static U32 g_time = 0;
/*****************************************
Local Parameters
*****************************************/
static char* programName;
static int displayLevel = 2;
/*********************************************************
Fuzzer functions
*********************************************************/
@ -114,7 +108,6 @@ static U32 FUZ_GetMilliStart(void)
return nCount;
}
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
U32 nCurrent = FUZ_GetMilliStart();
@ -124,9 +117,12 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart)
return nSpan;
}
static U32 FUZ_rotl32(U32 u32, U32 nbBits)
{
return ((u32 << nbBits) | (u32 >> (32 - nbBits)));
}
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src)
static U32 FUZ_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
@ -139,7 +135,7 @@ unsigned int FUZ_rand(unsigned int* src)
#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767)
#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed)
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed)
{
BYTE* BBuffer = (BYTE*)buffer;
int pos = 0;
@ -180,105 +176,103 @@ void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba,
#define BLOCKSIZE_I134 (32 MB)
static int FUZ_AddressOverflow(void)
{
char* buffers[MAX_NB_BUFF_I134+1] = {0};
int i, nbBuff=0;
int highAddress = 0;
char* buffers[MAX_NB_BUFF_I134+1] = {0};
int i, nbBuff=0;
int highAddress = 0;
printf("Overflow tests : ");
printf("Overflow tests : ");
// Only possible in 32-bits
if (sizeof(void*)==8)
{
printf("64 bits mode : no overflow \n");
fflush(stdout);
return 0;
}
buffers[0] = (char*)malloc(BLOCKSIZE_I134);
buffers[1] = (char*)malloc(BLOCKSIZE_I134);
if ((!buffers[0]) || (!buffers[1]))
{
printf("not enough memory for tests \n");
return 0;
}
for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++)
{
printf("%3i \b\b\b\b", nbBuff);
buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
//printf("%08X ", (U32)(size_t)(buffers[nbBuff]));
fflush(stdout);
if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress))
// Only possible in 32-bits
if (sizeof(void*)==8)
{
printf("high address detected : ");
printf("64 bits mode : no overflow \n");
fflush(stdout);
highAddress=1;
return 0;
}
if (buffers[nbBuff]==NULL) goto _endOfTests;
buffers[0] = (char*)malloc(BLOCKSIZE_I134);
buffers[1] = (char*)malloc(BLOCKSIZE_I134);
if ((!buffers[0]) || (!buffers[1]))
{
size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512);
int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
char* input = buffers[nbBuff-1];
char* output = buffers[nbBuff];
int r;
input[0] = (char)0xF0; // Literal length overflow
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
input[0] = (char)0x1F; // Match length overflow
input[1] = (char)0x01;
input[2] = (char)0x01;
input[3] = (char)0x00;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
output = buffers[nbBuff-2]; // Reverse in/out pointer order
input[0] = (char)0xF0; // Literal length overflow
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
input[0] = (char)0x1F; // Match length overflow
input[1] = (char)0x01;
input[2] = (char)0x01;
input[3] = (char)0x00;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
printf("not enough memory for tests \n");
return 0;
}
}
for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++)
{
printf("%3i \b\b\b\b", nbBuff);
buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
//printf("%08X ", (U32)(size_t)(buffers[nbBuff]));
fflush(stdout);
nbBuff++;
if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress))
{
printf("high address detected : ");
fflush(stdout);
highAddress=1;
}
if (buffers[nbBuff]==NULL) goto _endOfTests;
{
size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512);
int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
char* input = buffers[nbBuff-1];
char* output = buffers[nbBuff];
int r;
input[0] = (char)0xF0; // Literal length overflow
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
input[0] = (char)0x1F; // Match length overflow
input[1] = (char)0x01;
input[2] = (char)0x01;
input[3] = (char)0x00;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
output = buffers[nbBuff-2]; // Reverse in/out pointer order
input[0] = (char)0xF0; // Literal length overflow
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
input[0] = (char)0x1F; // Match length overflow
input[1] = (char)0x01;
input[2] = (char)0x01;
input[3] = (char)0x00;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
}
}
nbBuff++;
_endOfTests:
for (i=0 ; i<nbBuff; i++) free(buffers[i]);
if (!highAddress) printf("high address not possible \n");
else printf("all overflows correctly detected \n");
return 0;
for (i=0 ; i<nbBuff; i++) free(buffers[i]);
if (!highAddress) printf("high address not possible \n");
else printf("all overflows correctly detected \n");
return 0;
_overflowError:
printf("Address space overflow error !! \n");
exit(1);
printf("Address space overflow error !! \n");
exit(1);
}
static void FUZ_displayUpdate(int testNb)
static void FUZ_displayUpdate(unsigned testNb)
{
if ((FUZ_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=3))
if ((FUZ_GetMilliSpan(g_time) > g_refreshRate) || (g_displayLevel>=3))
{
g_time = FUZ_GetMilliStart();
DISPLAY("\r%5u ", testNb);
if (displayLevel>=3) fflush(stdout);
if (g_displayLevel>=3) fflush(stdout);
}
}
#define FUZ_MAX(a,b) (a>b?a:b)
static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility)
static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility)
{
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
@ -287,17 +281,20 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
unsigned int randState=seed;
int ret, cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; displayLevel<3 ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
int ret;
unsigned cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
LZ4_streamHC_t LZ4dictHC;
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
// init
@ -306,41 +303,48 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
// synd rand & dict
int dictSize, blockSize, blockStart;
char* dict;
char* block;
(void)FUZ_rand(&coreRandState);
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
if (0) // some problems related to dictionary re-use; in this case, enable this loop
{
int dictSize, blockSize, blockStart;
char* dict;
char* block;
FUZ_displayUpdate(cycleNb);
randState = coreRandState ^ PRIME3;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
}
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
U32 testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
FUZ_displayUpdate(cycleNb);
(void)FUZ_rand(&coreRandState);
randState = coreRandState ^ PRIME3;
// Select block to test
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
@ -488,20 +492,20 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[HCcompressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
/* Dictionary tests */
/* Dictionary tests */
// Compress using dictionary
FUZ_DISPLAYTEST;
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
free (LZ4continue);
@ -528,7 +532,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// Compress using External dictionary
FUZ_DISPLAYTEST;
dict -= 9; // Separation, so it is an ExtDict
dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
@ -551,13 +555,12 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
@ -566,7 +569,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
@ -582,15 +585,52 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
if (blockSize > 10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
U32 missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes)
{
decodedBuffer[blockSize-missingBytes] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
}
}
// Compress HC using External dictionary
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 7); // even bigger separation
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
// ***** End of tests *** //
// Fill stats
bytes += blockSize;
cbytes += compressedSize;
@ -598,7 +638,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("\r%7u /%7u - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
@ -606,7 +646,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// unalloc
{
int result = 0;
int result = 0;
_exit:
free(CNBuffer);
free(compressedBuffer);
@ -622,13 +662,320 @@ _output_error:
}
#define testInputSize (128 KB)
#define testCompressedSize (64 KB)
#define ringBufferSize (8 KB)
static void FUZ_unitTests(void)
{
const unsigned testNb = 0;
const unsigned seed = 0;
const unsigned cycleNb= 0;
char testInput[testInputSize];
char testCompressed[testCompressedSize];
char testVerify[testInputSize];
char ringBuffer[ringBufferSize];
U32 randState = 1;
// Init
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
// 32-bits address space overflow test
FUZ_AddressOverflow();
// LZ4 streaming tests
{
LZ4_stream_t* statePtr;
LZ4_stream_t streamingState;
U64 crcOrig;
U64 crcNew;
int result;
// Allocation test
statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr);
// simple compression test
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState);
result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// ring buffer test
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState;
const U32 maxMessageSizeLog = 10;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
U32 iNext = 0;
U32 rNext = 0;
U32 dNext = 0;
const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStream(&streamingState);
LZ4_setStreamDecode(&decodeState, NULL, 0);
while (iNext + messageSize < testCompressedSize)
{
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
result = LZ4_compress_limitedOutput_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = crcOrig = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// prepare next message
iNext += messageSize;
rNext += messageSize;
dNext += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
if (rNext + messageSize > ringBufferSize) rNext = 0;
if (dNext + messageSize > dBufferSize) dNext = 0;
}
}
}
// LZ4 HC streaming tests
{
LZ4_streamHC_t* sp;
LZ4_streamHC_t sHC;
//XXH64_state_t xxh;
U64 crcOrig;
U64 crcNew;
int result;
// Allocation test
sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp);
// simple compression test
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// simple dictionary compression test
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed");
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
// multiple HC compression test with dictionary
{
int result1, result2;
int segSize = testCompressedSize / 2;
crcOrig = XXH64(testInput + segSize, 64 KB, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, segSize);
result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1);
result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1);
FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2);
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize);
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
}
// remote dictionary HC compression test
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test");
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
// multiple HC compression with ext. dictionary
{
XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
const char* dict = testInput + 3;
int dictSize = (FUZ_rand(&randState) & 8191);
char* dst = testVerify;
size_t segStart = dictSize + 7;
int segSize = (FUZ_rand(&randState) & 8191);
int segNb = 1;
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, dict, dictSize);
XXH64_reset(&crcOrigState, 0);
XXH64_reset(&crcNewState, 0);
while (segStart + segSize < testInputSize)
{
XXH64_update(&crcOrigState, testInput + segStart, segSize);
crcOrig = XXH64_digest(&crcOrigState);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize));
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize);
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb);
XXH64_update(&crcNewState, dst, segSize);
crcNew = XXH64_digest(&crcNewState);
if (crcOrig!=crcNew)
{
size_t c=0;
while (dst[c] == testInput[segStart+c]) c++;
DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize);
}
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
dict = dst;
//dict = testInput + segStart;
dictSize = segSize;
dst += segSize + 1;
segNb ++;
segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1;
segSize = (FUZ_rand(&randState) & 8191);
}
}
// ring buffer test
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState;
const U32 maxMessageSizeLog = 10;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
U32 iNext = 0;
U32 rNext = 0;
U32 dNext = 0;
const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_setStreamDecode(&decodeState, NULL, 0);
while (iNext + messageSize < testCompressedSize)
{
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
result = LZ4_compressHC_limitedOutput_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = crcOrig = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// prepare next message
iNext += messageSize;
rNext += messageSize;
dNext += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
if (rNext + messageSize > ringBufferSize) rNext = 0;
if (dNext + messageSize > dBufferSize) dNext = 0;
}
}
// long stream test ; Warning : very long test !
if (1)
{
XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
const U64 totalTestSize = 6ULL << 30;
U64 totalTestDone = 0;
size_t oldStart = 0;
size_t oldSize = 0;
U32 segNb = 1;
DISPLAY("Long HC streaming test (%u MB)\n", (U32)(totalTestSize >> 20));
LZ4_resetStreamHC(&sHC, 0);
XXH64_reset(&crcOrigState, 0);
XXH64_reset(&crcNewState, 0);
while (totalTestDone < totalTestSize)
{
size_t testSize = (FUZ_rand(&randState) & 65535) + 1;
size_t testStart = FUZ_rand(&randState) & 65535;
FUZ_displayUpdate((U32)(totalTestDone >> 20));
XXH64_update(&crcOrigState, testInput + testStart, testSize);
crcOrig = XXH64_digest(&crcOrigState);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + testStart, testCompressed, (int)testSize, LZ4_compressBound((int)testSize));
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, (int)testSize, testInput + oldStart, (int)oldSize);
FUZ_CHECKTEST(result!=(int)testSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb);
XXH64_update(&crcNewState, testVerify, testSize);
crcNew = XXH64_digest(&crcNewState);
if (crcOrig!=crcNew)
{
size_t c=0;
while (testVerify[c] == testInput[testStart+c]) c++;
DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)testSize);
}
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
oldStart = testStart;
oldSize = testSize;
totalTestDone += testSize;
segNb ++;
}
DISPLAY("\r");
}
}
printf("All unit tests completed successfully \n");
return;
_output_error:
exit(1);
}
int FUZ_usage(void)
static int FUZ_usage(char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
@ -653,10 +1000,10 @@ int main(int argc, char** argv)
int nbTests = NB_ATTEMPTS;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int pause=0;
int pause = 0;
char* programName = argv[0];
// Check command line
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
@ -666,19 +1013,19 @@ int main(int argc, char** argv)
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; displayLevel=1; continue; }
if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; }
argument++;
while (argument[1]!=0)
while (*argument!=0)
{
argument++;
switch(*argument)
{
case 'h': /* display help */
return FUZ_usage();
return FUZ_usage(programName);
case 'v': /* verbose mode */
argument++;
displayLevel=4;
g_displayLevel=4;
break;
case 'p': /* pause at the end */
@ -744,7 +1091,7 @@ int main(int argc, char** argv)
printf("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
FUZ_unitTests();
if (seedset==0) FUZ_unitTests();
if (nbTests<=0) nbTests=1;