Merge pull request #6 from Cyan4973/streaming

Streaming
This commit is contained in:
Cyan4973 2014-06-11 23:07:25 +02:00
commit 2040c97c63
13 changed files with 1034 additions and 462 deletions

View File

@ -4,6 +4,7 @@ script: make test
before_install:
- sudo apt-get update -qq
- sudo apt-get install -qq gcc-multilib
- sudo apt-get install -qq valgrind
env:
- LZ4_TRAVIS_CI_ENV=-m32

View File

@ -92,7 +92,7 @@ lz4programs: lz4.c lz4hc.c
liblz4: lz4.c lz4hc.c
@echo compiling static library
@$(CC) $(CFLAGS) -c $^
@ar rcs liblz4.a lz4.o lz4hc.o
@$(AR) rcs liblz4.a lz4.o lz4hc.o
@echo compiling dynamic library
@$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@echo creating versioned links
@ -105,25 +105,25 @@ clean:
@echo Cleaning completed
#make install option is reserved to Linux & OSX targets
#make install option is designed for Linux & OSX targets only
ifneq (,$(filter $(shell uname),Linux Darwin))
install: liblz4
@install -d -m 755 $(DESTDIR)$(LIBDIR)/ $(DESTDIR)$(INCLUDEDIR)/
@install -m 755 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
@install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
@cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
@cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
@install -m 755 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@install -m 755 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
@install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@echo lz4 static and shared library installed
@cd $(PRGDIR); $(MAKE) -e install
uninstall:
[ -x $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
[ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h
[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@echo lz4 libraries successfully uninstalled

605
lz4.c Normal file → Executable file
View File

@ -34,15 +34,6 @@
/**************************************
Tuning parameters
**************************************/
/*
* MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#define MEMORY_USAGE 14
/*
* HEAPMODE :
* Select how default compression functions will allocate memory for their hash table,
@ -56,8 +47,9 @@
**************************************/
/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|| defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
|| defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
|| defined(__powerpc64__) || defined(__powerpc64le__) \
|| defined(__ppc64__) || defined(__ppc64le__) \
|| defined(__PPC64__) || defined(__PPC64LE__) \
|| defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
@ -68,6 +60,7 @@
* Little Endian or Big Endian ?
* Overwrite the #define below if you know your architecture endianess
*/
#include <stdlib.h> /* Apparently required to detect endianess */
#if defined (__GLIBC__)
# include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN)
@ -118,7 +111,6 @@
#endif
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# if LZ4_ARCH64 /* 64-bits */
# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */
@ -128,15 +120,6 @@
# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */
# endif
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#else
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
#endif
#ifdef _MSC_VER /* Visual Studio */
# define lz4_bswap16(x) _byteswap_ushort(x)
#else
# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
@ -224,9 +207,9 @@ typedef struct {size_t v;} _PACKED size_t_S;
/**************************************
Constants
**************************************/
#define LZ4_HASHLOG (MEMORY_USAGE-2)
#define HASHTABLESIZE (1 << MEMORY_USAGE)
#define HASHNBCELLS4 (1 << LZ4_HASHLOG)
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
#define HASH_SIZE_U32 (1 << LZ4_HASHLOG)
#define MINMATCH 4
@ -255,16 +238,18 @@ static const int LZ4_minLength = (MFLIMIT+1);
Structures and local types
**************************************/
typedef struct {
U32 hashTable[HASHNBCELLS4];
U32 hashTable[HASH_SIZE_U32];
U32 currentOffset;
U32 initCheck;
const BYTE* dictionary;
const BYTE* bufferStart;
const BYTE* base;
const BYTE* nextBlock;
} LZ4_Data_Structure;
U32 dictSize;
} LZ4_dict_t_internal;
typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive;
typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
typedef enum { byPtr, byU32, byU16 } tableType_t;
typedef enum { noDict = 0, withPrefix64k = 1, withExtDict=2 } dict_directive;
typedef enum { noDict = 0, withPrefix64k = 1, usingExtDict = 2 } dict_directive;
typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
@ -289,12 +274,12 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
/**************************************
Macros
**************************************/
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
#if LZ4_ARCH64 || !defined(__GNUC__)
# define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } /* at the end, d>=e; */
# define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } /* at the end, d>=e; */
#else
# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); }
# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); }
#endif
#define LZ4_SECURECOPY(d,s,e) { if (d<e) LZ4_WILDCOPY(d,s,e); }
/****************************
@ -302,7 +287,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
****************************/
#if LZ4_ARCH64
FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
int LZ4_NbCommonBytes (register U64 val)
{
# if defined(LZ4_BIG_ENDIAN)
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@ -334,7 +319,7 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
#else
FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
int LZ4_NbCommonBytes (register U32 val)
{
# if defined(LZ4_BIG_ENDIAN)
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@ -371,7 +356,7 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
****************************/
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
static int LZ4_hashSequence(U32 sequence, tableType_t tableType)
{
if (tableType == byU16)
return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
@ -379,9 +364,9 @@ FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
}
FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
static int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
switch (tableType)
{
@ -391,58 +376,96 @@ FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, t
}
}
FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
U32 h = LZ4_hashPosition(p, tableType);
LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
}
FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
{ U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
}
FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
U32 h = LZ4_hashPosition(p, tableType);
return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
}
static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
FORCE_INLINE int LZ4_compress_generic(
while (likely(pIn<pInLimit-(STEPSIZE-1)))
{
size_t diff = AARCH(pRef) ^ AARCH(pIn);
if (!diff) { pIn+=STEPSIZE; pRef+=STEPSIZE; continue; }
pIn += LZ4_NbCommonBytes(diff);
return (unsigned)(pIn - pStart);
}
if (LZ4_ARCH64) if ((pIn<(pInLimit-3)) && (A32(pRef) == A32(pIn))) { pIn+=4; pRef+=4; }
if ((pIn<(pInLimit-1)) && (A16(pRef) == A16(pIn))) { pIn+=2; pRef+=2; }
if ((pIn<pInLimit) && (*pRef == *pIn)) pIn++;
return (unsigned)(pIn - pStart);
}
static int LZ4_compress_generic(
void* ctx,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
limitedOutput_directive limitedOutput,
limitedOutput_directive outputLimited,
tableType_t tableType,
dict_directive dict)
{
LZ4_dict_t_internal* const dictPtr = (LZ4_dict_t_internal*)ctx;
const BYTE* ip = (const BYTE*) source;
const BYTE* const base = (dict==withPrefix64k) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source;
const BYTE* const lowLimit = ((dict==withPrefix64k) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source);
const BYTE* base;
const BYTE* lowLimit;
const BYTE* const dictionary = dictPtr->dictionary;
const BYTE* const dictEnd = dictionary + dictPtr->dictSize;
const size_t dictDelta = dictEnd - (const BYTE*)source;
const BYTE* anchor = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
BYTE* const olimit = op + maxOutputSize;
int length;
const int skipStrength = SKIPSTRENGTH;
U32 forwardH;
size_t refDelta=0;
/* Init conditions */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
if ((dict==withPrefix64k) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; /* must continue from end of previous block */
if (dict==withPrefix64k) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; /* do it now, due to potential early exit */
if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */
if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
switch(dict)
{
case noDict:
default:
base = (const BYTE*)source;
lowLimit = (const BYTE*)source;
break;
case withPrefix64k:
base = (const BYTE*)source - dictPtr->currentOffset;
lowLimit = (const BYTE*)source - dictPtr->dictSize;
if (lowLimit < base) lowLimit = base;
break;
case usingExtDict:
base = (const BYTE*)source - dictPtr->currentOffset;
lowLimit = (const BYTE*)source;
break;
}
if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */
if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
LZ4_putPosition(ip, ctx, tableType, base);
@ -451,98 +474,139 @@ FORCE_INLINE int LZ4_compress_generic(
/* Main Loop */
for ( ; ; )
{
int findMatchAttempts = (1U << skipStrength) + 3;
const BYTE* forwardIp = ip;
const BYTE* ref;
BYTE* token;
{
int step=1;
int searchMatchNb = (1U << skipStrength) + 3;
/* Find a match */
do {
U32 h = forwardH;
int step = findMatchAttempts++ >> skipStrength;
ip = forwardIp;
forwardIp = ip + step;
/* Find a match */
do {
U32 h = forwardH;
ip = forwardIp;
forwardIp += step;
step = searchMatchNb++ >> skipStrength;
if (unlikely (step>8)) step=8; // slows down uncompressible data; required for valid forwardIp
if (unlikely(forwardIp > mflimit)) { goto _last_literals; }
if (unlikely(ip > mflimit)) goto _last_literals;
forwardH = LZ4_hashPosition(forwardIp, tableType);
ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
if (dict==usingExtDict)
{
if (ref<(const BYTE*)source)
{
refDelta = dictDelta;
lowLimit = dictionary;
}
else
{
refDelta = 0;
lowLimit = (const BYTE*)source;
}
}
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
} while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip)));
} while ( ((tableType==byU16)? 0 : (ref + MAX_DISTANCE < ip)) || (A32(ref+refDelta) != A32(ip)) );
}
/* Catch up */
while ((ip>anchor) && (ref > lowLimit) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; }
/* Encode Literal length */
length = (int)(ip - anchor);
token = op++;
if ((limitedOutput) && (unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend))) return 0; /* Check output limit */
if (length>=(int)RUN_MASK)
{
int len = length-RUN_MASK;
*token=(RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
else *token = (BYTE)(length<<ML_BITS);
/* Encode Literal length */
unsigned litLength = (unsigned)(ip - anchor);
token = op++;
if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) return 0; /* Check output limit */
if (litLength>=RUN_MASK)
{
int len = (int)litLength-RUN_MASK;
*token=(RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
else *token = (BYTE)(litLength<<ML_BITS);
/* Copy Literals */
{ BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; }
/* Copy Literals */
{ BYTE* end=(op)+(litLength); LZ4_WILDCOPY(op,anchor,end); op=end; }
}
_next_match:
/* Encode Offset */
LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
/* Start Counting */
ip+=MINMATCH; ref+=MINMATCH; /* MinMatch already verified */
anchor = ip;
while (likely(ip<matchlimit-(STEPSIZE-1)))
{
size_t diff = AARCH(ref) ^ AARCH(ip);
if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
ip += LZ4_NbCommonBytes(diff);
goto _endCount;
}
if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
if ((ip<matchlimit) && (*ref == *ip)) ip++;
_endCount:
/* Encode MatchLength */
length = (int)(ip - anchor);
if ((limitedOutput) && (unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend))) return 0; /* Check output limit */
if (length>=(int)ML_MASK)
ref += refDelta;
{
*token += ML_MASK;
length -= ML_MASK;
for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; }
if (length >= 255) { length-=255; *op++ = 255; }
*op++ = (BYTE)length;
unsigned matchLength;
if ((dict==usingExtDict) && (lowLimit==dictionary))
{
const BYTE* limit = ip + (dictEnd-ref);
if (limit > matchlimit) limit = matchlimit;
matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit);
ip += MINMATCH + matchLength;
if (ip==limit)
{
unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit);
matchLength += more;
ip += more;
}
}
else
{
matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
ip += MINMATCH + matchLength;
}
if (matchLength>=ML_MASK)
{
if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) return 0; /* Check output limit */
*token += ML_MASK;
matchLength -= ML_MASK;
for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; }
if (matchLength >= 255) { matchLength-=255; *op++ = 255; }
*op++ = (BYTE)matchLength;
}
else *token += (BYTE)(matchLength);
}
else *token += (BYTE)(length);
anchor = ip;
/* Test end of chunk */
if (ip > mflimit) { anchor = ip; break; }
if (ip > mflimit) break;
/* Fill table */
LZ4_putPosition(ip-2, ctx, tableType, base);
/* Test next position */
ref = LZ4_getPosition(ip, ctx, tableType, base);
if (dict==usingExtDict)
{
if (ref<(const BYTE*)source)
{
refDelta = dictDelta;
lowLimit = dictionary;
}
else
{
refDelta = 0;
lowLimit = (const BYTE*)source;
}
}
LZ4_putPosition(ip, ctx, tableType, base);
if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
if ( (ref+MAX_DISTANCE>=ip) && (A32(ref+refDelta)==A32(ip)) ) { token=op++; *token=0; goto _next_match; }
/* Prepare next loop */
anchor = ip++;
forwardH = LZ4_hashPosition(ip, tableType);
forwardH = LZ4_hashPosition(++ip, tableType);
}
_last_literals:
/* Encode Last Literals */
{
int lastRun = (int)(iend - anchor);
if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
@ -557,9 +621,9 @@ _last_literals:
int LZ4_compress(const char* source, char* dest, int inputSize)
{
#if (HEAPMODE)
void* ctx = ALLOCATOR(HASHNBCELLS4, 4); /* Aligned on 4-bytes boundaries */
void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */
#else
U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
#endif
int result;
@ -577,16 +641,16 @@ int LZ4_compress(const char* source, char* dest, int inputSize)
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
{
#if (HEAPMODE)
void* ctx = ALLOCATOR(HASHNBCELLS4, 4); /* Aligned on 4-bytes boundaries */
void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */
#else
U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
#endif
int result;
if (inputSize < (int)LZ4_64KLIMIT)
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noDict);
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict);
else
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
#if (HEAPMODE)
FREEMEM(ctx);
@ -595,118 +659,169 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
}
/*****************************
Using external allocation
*****************************/
/*****************************************
Experimental : Streaming functions
*****************************************/
int LZ4_sizeofState() { return 1 << MEMORY_USAGE; }
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
void* LZ4_createStream()
{
if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_sizeofState());
if (inputSize < (int)LZ4_64KLIMIT)
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict);
else
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
void* lz4s = ALLOCATOR(4, LZ4_STREAMSIZE_U32);
MEM_INIT(lz4s, 0, LZ4_STREAMSIZE);
return lz4s;
}
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
int LZ4_free (void* LZ4_stream)
{
if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_sizeofState());
if (inputSize < (int)LZ4_64KLIMIT)
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, byU16, noDict);
else
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
}
/****************************
Stream functions
****************************/
int LZ4_sizeofStreamState()
{
return sizeof(LZ4_Data_Structure);
}
FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)
{
MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable));
lz4ds->bufferStart = base;
lz4ds->base = base;
lz4ds->nextBlock = base;
}
int LZ4_resetStreamState(void* state, const char* inputBuffer)
{
if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
LZ4_init((LZ4_Data_Structure*)state, (const BYTE*)inputBuffer);
return 0;
}
void* LZ4_create (const char* inputBuffer)
{
void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure));
LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer);
return lz4ds;
}
int LZ4_free (void* LZ4_Data)
{
FREEMEM(LZ4_Data);
FREEMEM(LZ4_stream);
return (0);
}
char* LZ4_slideInputBuffer (void* LZ4_Data)
int LZ4_loadDict (void* LZ4_dict, const char* dictionary, int dictSize)
{
LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data;
size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB);
LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
const BYTE* p = (const BYTE*)dictionary;
const BYTE* const dictEnd = p + dictSize;
const BYTE* base;
if ( (lz4ds->base - delta > lz4ds->base) /* underflow control */
|| ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) /* close to 32-bits limit */
LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_dict_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */
if (dict->initCheck) MEM_INIT(dict, 0, sizeof(LZ4_dict_t_internal));
if (dictSize < MINMATCH)
{
size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base;
int nH;
dict->dictionary = NULL;
dict->dictSize = 0;
return 1;
}
for (nH=0; nH < HASHNBCELLS4; nH++)
if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB;
base = p - dict->currentOffset;
dict->dictionary = p;
dict->dictSize = (U32)(dictEnd - p);
dict->currentOffset += dict->dictSize;
while (p <= dictEnd-MINMATCH)
{
LZ4_putPosition(p, dict, byU32, base);
p+=3;
}
return 1;
}
void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const BYTE* src)
{
if ((LZ4_dict->currentOffset > 0x80000000) ||
((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */
{
/* rescale hash table */
U32 delta = LZ4_dict->currentOffset - 64 KB;
int i;
for (i=0; i<HASH_SIZE_U32; i++)
{
if ((size_t)(lz4ds->hashTable[nH]) < deltaLimit) lz4ds->hashTable[nH] = 0;
else lz4ds->hashTable[nH] -= (U32)deltaLimit;
if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
else LZ4_dict->hashTable[i] -= delta;
}
memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
lz4ds->base = lz4ds->bufferStart;
lz4ds->nextBlock = lz4ds->base + 64 KB;
LZ4_dict->currentOffset = 64 KB;
LZ4_dict->dictionary = LZ4_dict->dictionary + LZ4_dict->dictSize - 64 KB;
LZ4_dict->dictSize = 64 KB;
}
else
}
int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize)
{
LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_stream;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
const BYTE* smallest = (const BYTE*) source;
if ((streamPtr->dictSize>0) && (smallest > dictEnd)) smallest = dictEnd;
LZ4_renormDictT(streamPtr, smallest);
if (dictEnd == (const BYTE*)source)
{
memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
lz4ds->nextBlock -= delta;
lz4ds->base -= delta;
int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
streamPtr->dictSize += (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
return (char*)(lz4ds->nextBlock);
{
int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
}
int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize)
int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_stream;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
const BYTE* smallest = (const BYTE*) source;
if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
LZ4_renormDictT(streamPtr, smallest);
if (dictEnd == (const BYTE*)source)
{
int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k);
streamPtr->dictSize += (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
{
int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
}
int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
// Hidden debug function, to force separate dictionary mode
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
{
return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix64k);
LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
int result;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
const BYTE* smallest = dictEnd;
if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
LZ4_renormDictT((LZ4_dict_t_internal*)LZ4_dict, smallest);
result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
int LZ4_moveDict (void* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
memcpy(safeBuffer, previousDictEnd - dictSize, dictSize);
dict->dictionary = (const BYTE*)safeBuffer;
dict->dictSize = (U32)dictSize;
return 1;
}
/****************************
Decompression functions
****************************/
@ -716,7 +831,7 @@ int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, cha
* Note that it is essential this generic function is really inlined,
* in order to remove useless branches during compilation optimisation.
*/
FORCE_INLINE int LZ4_decompress_generic(
static int LZ4_decompress_generic(
const char* source,
char* dest,
int inputSize,
@ -725,9 +840,9 @@ FORCE_INLINE int LZ4_decompress_generic(
int endOnInput, /* endOnOutputSize, endOnInputSize */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, withExtDict */
const char* dictStart, /* only if dict==withExtDict */
int dictSize /* only if dict==withExtDict */
int dict, /* noDict, withPrefix64k, usingExtDict */
const char* dictStart, /* only if dict==usingExtDict */
int dictSize /* only if dict==usingExtDict */
)
{
/* Local Variables */
@ -740,7 +855,7 @@ FORCE_INLINE int LZ4_decompress_generic(
BYTE* cpy;
BYTE* oexit = op + targetOutputSize;
const BYTE* const dictEnd = (dict==withExtDict) ? (const BYTE*)dictStart + dictSize : NULL;
const BYTE* const dictEnd = (dict==usingExtDict) ? (const BYTE*)dictStart + dictSize : NULL;
const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; /* static reduces speed for LZ4_decompress_safe() on GCC64 */
static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
@ -748,11 +863,10 @@ FORCE_INLINE int LZ4_decompress_generic(
/* Special cases */
(void)dictStart; (void)dictSize;
if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
/* Main Loop */
while (1)
{
@ -764,11 +878,7 @@ FORCE_INLINE int LZ4_decompress_generic(
if ((length=(token>>ML_BITS)) == RUN_MASK)
{
unsigned s=255;
while (((endOnInput)?ip<iend:1) && (s==255))
{
s = *ip++;
length += s;
}
while (((endOnInput)?ip<iend:1) && (s==255)) { s = *ip++; length += s; }
}
/* copy literals */
@ -814,7 +924,7 @@ FORCE_INLINE int LZ4_decompress_generic(
}
/* check external dictionary */
if ((dict==withExtDict) && (unlikely(ref < (BYTE* const)dest)))
if ((dict==usingExtDict) && (ref < (BYTE* const)dest))
{
if (unlikely(op+length+MINMATCH > oend-LASTLITERALS)) goto _output_error;
@ -830,7 +940,7 @@ FORCE_INLINE int LZ4_decompress_generic(
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
copySize = length+MINMATCH - copySize;
if (copySize > (size_t)((char*)op-dest))
if (copySize > (size_t)((char*)op-dest)) /* overlap */
{
BYTE* const cpy = op + copySize;
const BYTE* ref = (BYTE*)dest;
@ -862,7 +972,7 @@ FORCE_INLINE int LZ4_decompress_generic(
if (unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4)))
{
if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */
LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH));
if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, ref, (oend-COPYLENGTH));
while(op<cpy) *op++=*ref++;
op=cpy;
continue;
@ -895,7 +1005,7 @@ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compre
int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withExtDict, dictStart, dictSize);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
}
int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize)
@ -919,7 +1029,7 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withExtDict, dictStart, dictSize);
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize);
}
@ -927,8 +1037,71 @@ int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSi
Obsolete Functions
**************************************/
/*
These functions are deprecated and should no longer be used.
They are provided here for compatibility with existing user programs.
These function names are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
*/
int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
/* Obsolete Streaming functions */
int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
void LZ4_init(LZ4_dict_t_internal* lz4ds, const BYTE* base)
{
MEM_INIT(lz4ds->hashTable, 0, LZ4_STREAMSIZE);
lz4ds->bufferStart = base;
}
int LZ4_resetStreamState(void* state, const char* inputBuffer)
{
if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
LZ4_init((LZ4_dict_t_internal*)state, (const BYTE*)inputBuffer);
return 0;
}
void* LZ4_create (const char* inputBuffer)
{
void* lz4ds = ALLOCATOR(4, LZ4_STREAMSIZE_U32);
LZ4_init ((LZ4_dict_t_internal*)lz4ds, (const BYTE*)inputBuffer);
return lz4ds;
}
char* LZ4_slideInputBuffer (void* LZ4_Data)
{
LZ4_dict_t_internal* lz4ds = (LZ4_dict_t_internal*)LZ4_Data;
LZ4_moveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB);
return (char*)(lz4ds->bufferStart + 64 KB);
}
/* User-allocated state */
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
{
if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_sizeofState());
if (inputSize < (int)LZ4_64KLIMIT)
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict);
else
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
}
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
{
if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_sizeofState());
if (inputSize < (int)LZ4_64KLIMIT)
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict);
else
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
}

180
lz4.h
View File

@ -28,8 +28,8 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
@ -47,11 +47,16 @@ extern "C" {
/**************************************
Compiler Options
Tuning parameter
**************************************/
#if (defined(__GNUC__) && defined(__STRICT_ANSI__)) || (defined(_MSC_VER) && !defined(__cplusplus)) /* Visual Studio */
# define inline __inline /* Visual C is not C99, but supports some kind of inline */
#endif
/*
* LZ4_MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#define LZ4_MEMORY_USAGE 14
/**************************************
@ -72,10 +77,21 @@ LZ4_compress() :
or 0 if the compression fails
LZ4_decompress_safe() :
maxOutputSize : is the size of the destination buffer (which must be already allocated)
compressedSize : is obviously the source size
maxOutputSize : is the size of the destination buffer, which must be already allocated.
return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
If the destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
This function is protected against buffer overflow exploits :
it never writes outside of output buffer, and never reads outside of input buffer.
Therefore, it is protected against malicious data packets.
*/
/*
Note :
Should you prefer to explicitly allocate compression-table memory using your own allocation method,
use the streaming functions provided below, simply reset the memory area between each call to LZ4_compress_continue()
*/
@ -89,7 +105,6 @@ LZ4_decompress_safe() :
LZ4_compressBound() :
Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
primarily useful for memory allocation of output buffer.
inline function is recommended for the general case,
macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
@ -142,84 +157,70 @@ LZ4_decompress_safe_partial() :
int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize);
/*
These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
To know how much memory must be allocated for the compression tables, use :
int LZ4_sizeofState();
Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
The allocated memory can be provided to the compressions functions using 'void* state' parameter.
LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
They just use the externally allocated memory area instead of allocating their own one (on stack, or on heap).
*/
int LZ4_sizeofState(void);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
/**************************************
Streaming Functions
Experimental Streaming Functions
**************************************/
void* LZ4_create (const char* inputBuffer);
int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
char* LZ4_slideInputBuffer (void* LZ4_Data);
int LZ4_free (void* LZ4_Data);
#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U32 * sizeof(unsigned int))
/*
These functions allow the compression of chained blocks, where each block benefits from prior 64 KB within preceding blocks.
In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
void* LZ4_create (const char* inputBuffer);
The result of the function is the (void*) pointer on the LZ4 Data Structure.
This pointer will be needed in all other functions.
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
The input buffer must be already allocated, and size at least 192KB.
'inputBuffer' will also be the 'const char* source' of the first block.
All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
If next block does not begin immediately after the previous one, the compression will fail (return 0).
When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
char* LZ4_slideInputBuffer(void* LZ4_Data);
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
==> The memory position where the next input data block must start is provided as the result of the function.
Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
*/
* LZ4_stream_t
* information structure to track an LZ4 stream.
* set it to zero, or use LZ4_loadDict() to init it before first use.
*/
typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t;
/*
The following functions achieve the same result as :
void* LZ4_create (const char* inputBuffer);
* If you prefer dynamic allocation methods,
* LZ4_createStream
* provides a pointer (void*) towards an initialized LZ4_stream_t structure.
* LZ4_free just frees it.
*/
void* LZ4_createStream();
int LZ4_free (void* LZ4_stream);
They are provided here to allow the user program to allocate memory using its own routines.
/*
* LZ4_loadDict
* Use this function to load a static dictionary into LZ4_stream.
* Loading a size of 0 is allowed and init the LZ4_stream_t structure.
* Return : 1 if OK, 0 if error
*/
int LZ4_loadDict (void* LZ4_stream, const char* dictionary, int dictSize);
To know how much space must be allocated, use LZ4_sizeofStreamState();
Note also that space must be 4-bytes aligned.
/*
* LZ4_compress_continue
* Compress data block 'source', using blocks compressed before to improve compression ratio
* Previous data blocks are assumed to still be present at their previous location.
*/
int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize);
Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
void* state is a pointer to the space allocated.
It must be aligned on 4-bytes boundaries, and be large enough.
The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
The input buffer must be already allocated, and size at least 192KB.
'inputBuffer' will also be the 'const char* source' of the first block.
/*
* LZ4_compress_limitedOutput_continue
* Same as before, but also specify a maximum target compressed size (maxOutputSize)
* If it cannot be met, compression exits, and return a zero.
*/
int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
return value of LZ4_resetStreamState() must be 0 is OK.
Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
/*
* LZ4_moveDict
* If previously compressed data block is not guaranteed to remain at its previous memory location
* save it into a safe place (char* safeBuffer)
* before calling again LZ4_compress_continue()
* Return : 1 if OK, 0 if error
* Note : any dictSize > 64 KB will be interpreted as 64KB.
*/
int LZ4_moveDict (void* LZ4_stream, char* safeBuffer, int dictSize);
/*
*_usingDict() :
These decoding functions work the same as their "normal" versions,
but can also use up to 64KB of dictionary data (dictStart, dictSize)
to decode chained blocks.
*/
int LZ4_sizeofStreamState(void);
int LZ4_resetStreamState(void* state, const char* inputBuffer);
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
/*
*_withPrefix64k() :
@ -232,30 +233,31 @@ int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compr
int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
/**************************************
Experimental Functions
**************************************/
/*
*_withDict() :
These decoding functions work the same as their "normal" versions,
but can also use up to 64KB of dictionary data
to decode chained blocks.
*/
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
/**************************************
Obsolete Functions
**************************************/
/*
These functions are deprecated and should no longer be used.
They are provided here for compatibility with existing user programs.
Obsolete decompression functions
These function names are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
- LZ4_uncompress is the same as LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
*/
int LZ4_uncompress (const char* source, char* dest, int outputSize);
int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/* Obsolete external allocation functions */
int LZ4_sizeofState(void);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete streaming functions */
void* LZ4_create (const char* inputBuffer);
int LZ4_sizeofStreamState(void);
int LZ4_resetStreamState(void* state, const char* inputBuffer);
char* LZ4_slideInputBuffer (void* state);
#if defined (__cplusplus)
}

View File

@ -54,8 +54,9 @@
**************************************/
/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|| defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
|| defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
|| defined(__powerpc64__) || defined(__powerpc64le__) \
|| defined(__ppc64__) || defined(__ppc64le__) \
|| defined(__PPC64__) || defined(__PPC64LE__) \
|| defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
@ -66,6 +67,7 @@
* Little Endian or Big Endian ?
* Overwrite the #define below if you know your architecture endianess
*/
#include <stdlib.h> /* Apparently required to detect endianess */
#if defined (__GLIBC__)
# include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN)

View File

@ -104,6 +104,7 @@ They just use the externally allocated memory area instead of allocating their o
/**************************************
Streaming Functions
**************************************/
/* Note : these streaming functions still follows the older model */
void* LZ4_createHC (const char* inputBuffer);
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);

View File

@ -42,27 +42,32 @@ MANDIR=$(PREFIX)/share/man/man1
LZ4DIR=..
TEST_FILES = COPYING
TEST_TARGETS=test-32 test-64
TEST_TARGETS=test-64 test-32
BENCH_NB=-i5
# Minimize test target for Travis CI's Build Matrix
ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
TEST_TARGETS=test-32
BENCH_NB=-i1
else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
TEST_TARGETS=test-64
BENCH_NB=-i1
endif
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
VOID = nul
else
EXT =
VOID = /dev/null
endif
default: lz4 lz4c
all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32
all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 datagen
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
@ -73,22 +78,27 @@ lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
datagen : datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
@rm -f core *.o \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT)
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) datagen$(EXT)
@echo Cleaning completed
@ -117,21 +127,29 @@ uninstall:
test: $(TEST_TARGETS)
test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32
test-64: test-lz4 test-lz4c test-fullbench test-fuzzer test-mem
test-64: test-lz4 test-lz4c test-fullbench test-fuzzer
test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32 test-mem32
test-lz4:
test-lz4: lz4 datagen
./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
test-lz4c:
test-lz4c: lz4c datagen
test-lz4c32:
test-lz4c32: lz4c32 lz4 datagen
./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID)
./datagen | ./lz4c32 | ./lz4 -vdq > $(VOID)
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
test-fullbench: fullbench
./fullbench --no-prompt $(TEST_FILES)
./fullbench --no-prompt $(BENCH_NB) $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(TEST_FILES)
./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES)
test-fuzzer: fuzzer
./fuzzer --no-prompt
@ -139,5 +157,14 @@ test-fuzzer: fuzzer
test-fuzzer32: fuzzer32
./fuzzer32 --no-prompt
test-mem: lz4 datagen
./datagen -g256MB > tmp
valgrind ./lz4 -B4D -f tmp /dev/null
./datagen -g16MB > tmp
valgrind ./lz4 -9 -B5D -f tmp /dev/null
rm tmp
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
endif

283
programs/datagen.c Normal file
View File

@ -0,0 +1,283 @@
/*
datagen.c - compressible data generator test tool
Copyright (C) Yann Collet 2012-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
/**************************************
Remove Visual warning messages
**************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets
/**************************************
Includes
**************************************/
//#include <stdlib.h>
#include <stdio.h> // fgets, sscanf
#include <string.h> // strcmp
/**************************************
Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
Constants
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION "rc118"
#endif
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
#define CDG_SIZE_DEFAULT (64 KB)
#define CDG_SEED_DEFAULT 0
#define CDG_COMPRESSIBILITY_DEFAULT 50
#define PRIME1 2654435761U
#define PRIME2 2246822519U
/**************************************
Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/**************************************
Local Parameters
**************************************/
static int no_prompt = 0;
static char* programName;
static int displayLevel = 2;
/*********************************************************
Fuzzer functions
*********************************************************/
#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int CDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 = CDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
}
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0')
static void CDG_generate(U64 size, U32* seed, double proba)
{
BYTE fullbuff[32 KB + 128 KB + 1];
BYTE* buff = fullbuff + 32 KB;
U64 total=0;
U32 P32 = (U32)(32768 * proba);
U32 pos=0;
U32 genBlockSize = 128 KB;
// Build initial prefix
while (pos<32 KB)
{
// Select : Literal (char) or Match (within 32K)
if (CDG_RAND15BITS < P32)
{
// Copy (within 64K)
U32 d;
int ref;
int length = CDG_RANDLENGTH + 4;
U32 offset = CDG_RAND15BITS + 1;
if (offset > pos) offset = pos;
ref = pos - offset;
d = pos + length;
while (pos < d) fullbuff[pos++] = fullbuff[ref++];
}
else
{
// Literal (noise)
U32 d;
int length = CDG_RANDLENGTH;
d = pos + length;
while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
}
}
// Generate compressible data
pos = 0;
while (total < size)
{
if (size-total < 128 KB) genBlockSize = (U32)(size-total);
total += genBlockSize;
buff[genBlockSize] = 0;
pos = 0;
while (pos<genBlockSize)
{
// Select : Literal (char) or Match (within 32K)
if (CDG_RAND15BITS < P32)
{
// Copy (within 64K)
int ref;
U32 d;
int length = CDG_RANDLENGTH + 4;
U32 offset = CDG_RAND15BITS + 1;
if (pos + length > genBlockSize ) length = genBlockSize - pos;
ref = pos - offset;
d = pos + length;
while (pos < d) buff[pos++] = buff[ref++];
}
else
{
// Literal (noise)
U32 d;
int length = CDG_RANDLENGTH;
if (pos + length > genBlockSize) length = genBlockSize - pos;
d = pos + length;
while (pos < d) buff[pos++] = CDG_RANDCHAR;
}
}
pos=0;
for (;pos+512<=genBlockSize;pos+=512) printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
// Regenerate prefix
memcpy(fullbuff, buff + 96 KB, 32 KB);
}
}
int CDG_usage()
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
DISPLAY( " %s [size] [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, char** argv)
{
int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT;
U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT;
// Check command line
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
while (argument[1]!=0)
{
argument++;
switch(*argument)
{
case 'h':
return CDG_usage();
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
{
size *= 10;
size += *argument - '0';
argument++;
}
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 'p':
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
case 'v':
displayLevel = 4;
break;
default: ;
}
}
}
}
// Get Seed
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
CDG_generate(size, &seed, ((double)proba) / 100);
return 0;
}

127
programs/fullbench.c Normal file → Executable file
View File

@ -254,88 +254,104 @@ static U64 BMK_GetFileSize(char* infilename)
Benchmark function
*********************************************************/
static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
}
static void* stateLZ4;
static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize)
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_withState(stateLZ4, in, out, inSize);
}
static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
}
static void* ctx;
static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize)
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_continue(ctx, in, out, inSize);
}
static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
}
LZ4_stream_t LZ4_dict;
static void* local_LZ4_resetDictT(const char* fake)
{
(void)fake;
memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
return NULL;
}
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
{
return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
}
static void* stateLZ4HC;
static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
}
static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
}
static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
}
static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_continue(ctx, in, out, inSize);
}
static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
}
static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_fast(in, out, outSize);
return outSize;
}
static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_fast_withPrefix64k(in, out, outSize);
return outSize;
}
static inline int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_fast_usingDict(in, out, outSize, in - 65536, 65536);
return outSize;
}
static inline int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_safe_usingDict(in, out, inSize, outSize, in - 65536, 65536);
return outSize;
}
static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
{
return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
}
@ -344,23 +360,16 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
{
int fileIdx=0;
char* orig_buff;
# define NB_COMPRESSION_ALGORITHMS 12
# define NB_COMPRESSION_ALGORITHMS 13
# define MINCOMPRESSIONCHAR '0'
# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput",
"LZ4_compress_withState", "LZ4_compress_limitedOutput_withState",
"LZ4_compress_continue", "LZ4_compress_limitedOutput_continue",
"LZ4_compressHC", "LZ4_compressHC_limitedOutput",
"LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC",
"LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
# define NB_DECOMPRESSION_ALGORITHMS 7
# define MINDECOMPRESSIONCHAR '0'
# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)
static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_fast_usingDict",
"LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_usingDict", "LZ4_decompress_safe_partial" };
double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};
double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
U64 totals = 0;
@ -465,29 +474,30 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
DISPLAY(" %s : \n", inFileName);
// Compression Algorithms
for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
{
char* cName = compressionNames[cAlgNb];
char* compressorName;
int (*compressionFunction)(const char*, char*, int);
void* (*initFunction)(const char*) = NULL;
double bestTime = 100000000.;
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb+1)) continue;
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
switch(cAlgNb)
{
case 0 : compressionFunction = LZ4_compress; break;
case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break;
case 2 : compressionFunction = local_LZ4_compress_withState; break;
case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break;
case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;
case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;
case 6 : compressionFunction = LZ4_compressHC; break;
case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break;
case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break;
case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break;
case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;
case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;
case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
}
@ -496,7 +506,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
double averageTime;
int milliTime;
PROGRESS("%1i-%-21.21s : %9i ->\r", loopNb, cName, (int)benchedSize);
PROGRESS("%1i-%-25.25s : %9i ->\r", loopNb, compressorName, (int)benchedSize);
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
nb_loops = 0;
@ -509,7 +519,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
}
if (initFunction!=NULL) free(ctx);
nb_loops++;
@ -520,13 +530,13 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
if (averageTime < bestTime) bestTime = averageTime;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
PROGRESS("%1i-%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
PROGRESS("%1i-%-25.25s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
}
if (ratio<100.)
DISPLAY("%-23.23s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
DISPLAY("%-27.27s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
else
DISPLAY("%-23.23s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
DISPLAY("%-27.27s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
totalCTime[cAlgNb] += bestTime;
totalCSize[cAlgNb] += cSize;
@ -536,7 +546,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{
chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
}
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
@ -558,7 +568,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
case 4: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
case 5: decompressionFunction = local_LZ4_decompress_safe_usingDict; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_partial; break;
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
}
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
@ -607,25 +617,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
free(chunkP);
}
if (nbFiles > 1)
{
int AlgNb;
DISPLAY(" ** TOTAL ** : \n");
for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
{
char* cName = compressionNames[AlgNb];
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
DISPLAY("%-23.23s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
}
for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
{
char* dName = decompressionNames[AlgNb];
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
DISPLAY("%-31.31s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
}
}
if (BMK_pause) { printf("press enter...\n"); getchar(); }
return 0;
@ -669,7 +660,7 @@ int main(int argc, char** argv)
char* input_filename=0;
// Welcome message
DISPLAY( WELCOME_MESSAGE);
DISPLAY(WELCOME_MESSAGE);
if (argc<2) { badusage(exename); return 1; }

View File

@ -83,11 +83,12 @@
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
//**************************************
// Local Parameters
//**************************************
/*****************************************
Local Parameters
*****************************************/
static int no_prompt = 0;
static char* programName;
static int displayLevel = 2;
/*********************************************************
@ -187,7 +188,7 @@ int FUZ_SecurityTest()
#define FUZ_MAX(a,b) (a>b?a:b)
int FUZ_test(U32 seed, int nbTests, double compressibility) {
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
unsigned long long hcbytes = 0;
@ -197,14 +198,16 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
unsigned int randState=seed;
int ret, attemptNb;
int ret, cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %i) \n", seed, attemptNb); goto _output_error; }
# define FUZ_DISPLAYTEST testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
U32 crcOrig, crcCheck;
int displayRefresh;
// Create compressible test buffer
@ -213,8 +216,25 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// display refresh rate
switch(displayLevel)
{
case 0: displayRefresh = nbCycles+1; break;
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
case 2: displayRefresh=99; break;
default : displayRefresh=1;
}
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
FUZ_rand(&randState);
FUZ_rand(&randState);
FUZ_rand(&randState);
}
// Test loop
for (attemptNb = 0; attemptNb < nbTests; attemptNb++)
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
char* dict;
@ -222,14 +242,10 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
// note : promptThrottle is throtting stdout to prevent
// Travis-CI's output limit (10MB) and false hangup detection.
const int step = FUZ_MAX(1, nbTests / 100);
const int promptThrottle = ((attemptNb % step) == 0);
if (!no_prompt || attemptNb == 0 || promptThrottle)
if ((cycleNb%displayRefresh) == 0)
{
printf("\r%7i /%7i - ", attemptNb, nbTests);
if (no_prompt) fflush(stdout);
printf("\r%7i /%7i - ", cycleNb, nbCycles);
fflush(stdout);
}
// Select block to test
@ -305,7 +321,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than target size"); // well, is that an issue ?
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
@ -401,7 +417,14 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data");
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
@ -409,6 +432,27 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
// Compress using dictionary
FUZ_DISPLAYTEST;
dict -= 9;
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed");
FUZ_DISPLAYTEST;
memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_usingDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_usingDict should work : enough size available within output buffer");
// Decompress with dictionary as external
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
@ -416,7 +460,14 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data");
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
@ -455,7 +506,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", attemptNb, nbTests);
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
@ -489,7 +540,9 @@ int FUZ_usage()
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -v : verbose\n");
DISPLAY( " -h : display help and exit\n");
return 0;
}
@ -502,6 +555,7 @@ int main(int argc, char** argv) {
int seedset=0;
int argNb;
int nbTests = NB_ATTEMPTS;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
// Check command line
@ -515,7 +569,7 @@ int main(int argc, char** argv) {
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; continue; }
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }
while (argument[1]!=0)
{
@ -524,6 +578,10 @@ int main(int argc, char** argv) {
{
case 'h':
return FUZ_usage();
case 'v':
argument++;
displayLevel=4;
break;
case 'i':
argument++;
nbTests=0;
@ -544,6 +602,16 @@ int main(int argc, char** argv) {
argument++;
}
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9'))
{
testNb *= 10;
testNb += *argument - '0';
argument++;
}
break;
case 'p':
argument++;
proba=0;
@ -583,5 +651,5 @@ int main(int argc, char** argv) {
if (nbTests<=0) nbTests=1;
return FUZ_test(seed, nbTests, ((double)proba) / 100);
return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
}

View File

@ -64,6 +64,7 @@ following options
.TP
.B \-B#
block size [4-7](default : 7)
B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
.TP
.B \-BD
block dependency (improve compression ratio)
@ -84,4 +85,4 @@ following options
Report bugs at:- https://code.google.com/p/lz4/
.SH AUTHOR
Yann Collet
Yann Collet

View File

@ -109,7 +109,7 @@
//****************************
#define COMPRESSOR_NAME "LZ4 Compression CLI"
#ifndef LZ4_VERSION
# define LZ4_VERSION "v1.1.5"
# define LZ4_VERSION "v1.1.8"
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
@ -344,6 +344,19 @@ int main(int argc, char** argv)
if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)
#endif // DISABLE_LZ4C_LEGACY_OPTIONS
if ((*argument>='0') && (*argument<='9'))
{
cLevel = 0;
while ((*argument >= '0') && (*argument <= '9'))
{
cLevel *= 10;
cLevel += *argument - '0';
argument++;
}
argument--;
continue;
}
switch(argument[0])
{
// Display help
@ -354,20 +367,6 @@ int main(int argc, char** argv)
// Compression (default)
case 'z': forceCompress = 1; break;
// Compression level
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'A': /* non documented (hidden) */
cLevel=*argument -'0'; break;
// Use Legacy format (for Linux kernel compression)
case 'l': legacy_format=1; break;
@ -405,9 +404,8 @@ int main(int argc, char** argv)
case '7':
{
int B = argument[1] - '0';
int S = 1 << (8 + 2*B);
BMK_SetBlocksize(S);
blockSize = LZ4IO_setBlockSizeID(B);
BMK_SetBlocksize(blockSize);
argument++;
break;
}

View File

@ -365,17 +365,37 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i
}
static void* LZ4IO_LZ4_createStream (const char* inputBuffer)
{
(void)inputBuffer;
return LZ4_createStream();
}
static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
(void)compressionLevel;
return LZ4_compress_limitedOutput_continue(ctx, source, dest, inputSize, maxOutputSize);
}
static int LZ4IO_LZ4_slideInputBufferHC (void* ctx, char* buffer, int size)
{
(void)size; (void)buffer;
LZ4_slideInputBufferHC (ctx);
return 1;
}
static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
{
void* (*initFunction) (const char*);
int (*compressionFunction)(void*, const char*, char*, int, int);
char* (*translateFunction) (void*);
int (*compressionFunction)(void*, const char*, char*, int, int, int);
int (*nextBlockFunction) (void*, char*, int);
int (*freeFunction) (void*);
void* ctx;
unsigned long long filesize = 0;
unsigned long long compressedfilesize = 0;
unsigned int checkbits;
char* in_buff, *in_start, *in_end;
char* in_buff, *in_blockStart;
char* out_buff;
FILE* finput;
FILE* foutput;
@ -384,34 +404,35 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
size_t sizeCheck, header_size;
void* streamChecksumState=NULL;
// Init
start = clock();
if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
if (compressionlevel>=3)
if (compressionlevel<3)
{
initFunction = LZ4_createHC;
compressionFunction = LZ4_compressHC_limitedOutput_continue;
translateFunction = LZ4_slideInputBufferHC;
freeFunction = LZ4_freeHC;
initFunction = LZ4IO_LZ4_createStream;
compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue;
nextBlockFunction = LZ4_moveDict;
freeFunction = LZ4_free;
}
else
{
initFunction = LZ4_create;
compressionFunction = LZ4_compress_limitedOutput_continue;
translateFunction = LZ4_slideInputBuffer;
initFunction = LZ4_createHC;
compressionFunction = LZ4_compressHC2_limitedOutput_continue;
nextBlockFunction = LZ4IO_LZ4_slideInputBufferHC;
freeFunction = LZ4_free;
}
get_fileHandle(input_filename, output_filename, &finput, &foutput);
blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
// Allocate Memory
inputBufferSize = blockSize + 64 KB;
if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
inputBufferSize = 64 KB + blockSize;
in_buff = (char*)malloc(inputBufferSize);
out_buff = (char*)malloc(blockSize+CACHELINE);
if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
in_start = in_buff; in_end = in_buff + inputBufferSize;
in_blockStart = in_buff + 64 KB;
if (compressionlevel>=3) in_blockStart = in_buff;
if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
ctx = initFunction(in_buff);
@ -435,19 +456,19 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
{
unsigned int outSize;
unsigned int inSize;
// Read Block
if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
inSize = (unsigned int) fread(in_blockStart, (size_t)1, (size_t)blockSize, finput);
if( inSize==0 ) break; // No more input : end of compression
filesize += inSize;
DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
if (streamChecksum) XXH32_update(streamChecksumState, in_blockStart, inSize);
// Compress Block
outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel);
if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
if (blockChecksum) compressedfilesize+=4;
DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100);
// Write Block
if (outSize > 0)
@ -462,24 +483,28 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
sizeToWrite = 4 + outSize + (4*blockChecksum);
sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
}
else // Copy Original
{
* (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
sizeCheck = fwrite(out_buff, 1, 4, foutput);
if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
sizeCheck = fwrite(in_start, 1, inSize, foutput);
sizeCheck = fwrite(in_blockStart, 1, inSize, foutput);
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
if (blockChecksum)
{
unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
unsigned int checksum = XXH32(in_blockStart, inSize, LZ4S_CHECKSUM_SEED);
* (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
sizeCheck = fwrite(out_buff, 1, 4, foutput);
if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
}
}
in_start += inSize;
{
size_t sizeToMove = 64 KB;
if (inSize < 64 KB) sizeToMove = inSize;
nextBlockFunction(ctx, in_blockStart - sizeToMove, sizeToMove);
if (compressionlevel>=3) in_blockStart = in_buff + 64 KB;
}
}
// End of Stream mark
@ -542,7 +567,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
// Init
start = clock();
if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local;
if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local;
else { compressionFunction = LZ4_compressHC2_limitedOutput; }
get_fileHandle(input_filename, output_filename, &finput, &foutput);
blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
@ -587,7 +612,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel);
if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
if (blockChecksum) compressedfilesize+=4;
DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100);
// Write Block
if (outSize > 0)