AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

- New cmake file, by Nobuhiro Iwamatsu, which can also produce shared and static libraries.

Browse Source 
- Improved decoding speed, even more for 64-bits, and "safe" version
- Slight speed increase for LZ4-HC
- Pushed a useless parameter down the list in lz4.c

git-svn-id: https://lz4.googlecode.com/svn/trunk@90 650e7d94-2a16-8b24-b05c-7c0b3f6821cd
This commit is contained in:
yann.collet.73@gmail.com 2013-03-02 23:34:21 +00:00
parent e898c9a79a
commit 633c51904e
6 changed files with 184 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bench.c
View File

@ -433,7 +433,8 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
chunkP[chunkNb].outputSize = compP.decompressionFunction(chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputBuffer, chunkP[chunkNb].inputSize);
chunkP[chunkNb].outputSize = LZ4_uncompress(chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputBuffer, chunkP[chunkNb].inputSize);
//chunkP[chunkNb].inputSize = LZ4_uncompress_unknownOutputSize(chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputBuffer, chunkP[chunkNb].outputSize, chunkSize);
nb_loops++;
}
milliTime = BMK_GetMilliSpan(milliTime);

72
cmake/CMakeLists.txt
View File

@ -4,7 +4,7 @@ set(CPACK_PACKAGE_VERSION_MAJOR 0)
set(CPACK_PACKAGE_VERSION_MINOR 0)
set(CPACK_PACKAGE_VERSION_PATCH r51)
#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL)
set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ")
set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ")
include(CPack)
@ -16,25 +16,75 @@ IF( ${SIZEOF_VOID_P} STREQUAL "8" )
MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P})
ENDIF()
set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")
if (BUILD_SHARED_LIBS)
message(STATUS "Enable shared library building")
else(BUILD_SHARED_LIBS)
message(STATUS "Disable shared library building")
endif(BUILD_SHARED_LIBS)
if(UNIX AND BUILD_SHARED_LIBS)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
add_definitions(-fPIC)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
endif()
set(SRC_DIR ../)
set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h )
set(LZ4_SRCS ${LZ4_SRCS_LIB} ${SRC_DIR}bench.c ${SRC_DIR}lz4demo.c )
set(LZ4_SRCS ${SRC_DIR}bench.c ${SRC_DIR}lz4demo.c)
# EXECUTABLES FOR 32 Bit and 64 versions
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")
add_executable(lz4demo64 ${LZ4_SRCS})
install(TARGETS lz4demo64 RUNTIME DESTINATION "./")
if(NOT BUILD_SHARED_LIBS)
set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB})
endif()
add_executable(lz4demo32 ${LZ4_SRCS})
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")
message(STATUS "Build 64bit executable binary")
add_executable(lz4demo64 ${LZ4_SRCS})
install(TARGETS lz4demo64 RUNTIME DESTINATION "./")
if(NOT BUILD_SHARED_LIBS)
message(STATUS "Build 32bit executable binary")
add_executable(lz4demo32 ${LZ4_SRCS})
install(TARGETS lz4demo32 RUNTIME DESTINATION "./")
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")
SET_TARGET_PROPERTIES(lz4demo32 PROPERTIES
COMPILE_FLAGS PROPERTIES COMPILE_FLAGS "-m32" LINK_FLAGS "-m32")
SET_TARGET_PROPERTIES(lz4demo32 PROPERTIES
COMPILE_FLAGS PROPERTIES COMPILE_FLAGS "-m32" LINK_FLAGS "-m32")
endif()
else()
message(STATUS "Build 32bit executable binary")
add_executable(lz4demo32 ${LZ4_SRCS})
install(TARGETS lz4demo32 RUNTIME DESTINATION "./")
endif()
if(BUILD_SHARED_LIBS)
set(LZ4_SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}")
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")
target_link_libraries(lz4demo64 liblz4)
else()
target_link_libraries(lz4demo32 liblz4)
endif()
# for static library
add_library(liblz4_static STATIC ${LZ4_SRCS_LIB})
set_target_properties(liblz4_static PROPERTIES OUTPUT_NAME lz4)
install(TARGETS liblz4_static
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib
)
# for shared library o
add_library(liblz4 ${LZ4_SRCS_LIB})
set_target_properties(liblz4 PROPERTIES
OUTPUT_NAME lz4
SOVERSION ${LZ4_SOVERSION})
install(TARGETS liblz4
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib
)
endif(BUILD_SHARED_LIBS)
#warnings
@ -47,6 +97,4 @@ ADD_DEFINITIONS("-Os -march=native -std=c99")
INCLUDE_DIRECTORIES (${SRC_DIR})
#target_link_libraries(lz4 ${LZ4_SRCS_LIB})

21
cmake/release_COPYING.txt Normal file
View File

@ -0,0 +1,21 @@
lz4demo and fuzzer is an open-source demo compression algorithm LZ4 programs
Copyright (C) Yann Collet 2012
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/

7
fuzzer.c
View File

@ -144,7 +144,8 @@ int main() {
for (i = 0; i < 2048; i++)
cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&seed) >> 16;
for (i = 0; i < NB_ATTEMPTS; i++) {
for (i = 0; i < NB_ATTEMPTS; i++)
{
printf("\r%7i /%7i\r", i, NB_ATTEMPTS);
FUZ_rand(&seed);
@ -171,7 +172,7 @@ int main() {
// Test decoding with output size being exactly what's necessary => must work
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN);
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
if (ret<0) { printf("decompression failed despite correct space: seed %u, len %d\n", seed, LEN); goto _output_error; }
// Test decoding with one byte missing => must fail
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1);
@ -185,7 +186,7 @@ int main() {
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
// Test decoding with output size being exactly what's necessary => should work
// Test decoding with output size being exactly what's necessary => must work
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN);
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }

98
lz4.c
View File

@ -41,14 +41,6 @@
// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
#define MEMORY_USAGE 14
// NOTCOMPRESSIBLE_DETECTIONLEVEL :
// Decreasing this value will make the algorithm skip faster data segments considered "incompressible"
// This may decrease compression ratio dramatically, but will be faster on incompressible data
// Increasing this value will make the algorithm search more before declaring a segment "incompressible"
// This could improve compression a bit, but will be slower on incompressible data
// The default value (6) is recommended
#define NOTCOMPRESSIBLE_DETECTIONLEVEL 6
// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
// This will provide a small boost to performance for big endian cpu, but the resulting compressed stream will be incompatible with little-endian CPU.
// You can set this option to 1 in situations where data will remain within closed environment
@ -188,6 +180,13 @@ typedef struct _U64_S { U64 v; } U64_S;
#define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1)
// NOTCOMPRESSIBLE_DETECTIONLEVEL :
// Decreasing this value will make the algorithm skip faster data segments considered "incompressible"
// This may decrease compression ratio dramatically, but will be faster on incompressible data
// Increasing this value will make the algorithm search more before declaring a segment "incompressible"
// This could improve compression a bit, but will be slower on incompressible data
// The default value (6) is recommended
#define NOTCOMPRESSIBLE_DETECTIONLEVEL 6
#define SKIPSTRENGTH (NOTCOMPRESSIBLE_DETECTIONLEVEL>2?NOTCOMPRESSIBLE_DETECTIONLEVEL:2)
#define STACKLIMIT 13
#define HEAPMODE (HASH_LOG>STACKLIMIT) // Defines if memory is allocated into the stack (local variable), or into the heap (malloc()).
@ -309,7 +308,7 @@ static inline int LZ4_NbCommonBytes (register U32 val)
#endif
#else
#if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
unsigned long r;
_BitScanForward( &r, val );
return (int)(r>>3);
#elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
@ -358,7 +357,7 @@ static inline int LZ4_compressCtx(void** ctx,
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
int len, length;
int length;
const int skipStrength = SKIPSTRENGTH;
U32 forwardH;
@ -430,7 +429,14 @@ static inline int LZ4_compressCtx(void** ctx,
}
else *token = (length<<ML_BITS);
#else
if (length>=(int)RUN_MASK) { *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; }
if (length>=(int)RUN_MASK)
{
int len;
*token=(RUN_MASK<<ML_BITS);
len = length-RUN_MASK;
for(; len > 254 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
else *token = (length<<ML_BITS);
#endif
@ -442,7 +448,7 @@ _next_match:
LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
// Start Counting
ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified
ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified
anchor = ip;
while likely(ip<matchlimit-(STEPSIZE-1))
{
@ -457,10 +463,17 @@ _next_match:
_endCount:
// Encode MatchLength
len = (int)(ip - anchor);
if unlikely(op + (1 + LASTLITERALS) + (len>>8) > oend) return 0; // Check output limit
if (len>=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; }
else *token += len;
length = (int)(ip - anchor);
if unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend) return 0; // Check output limit
if (length>=(int)ML_MASK)
{
*token += ML_MASK;
length -= ML_MASK;
for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; }
if (length > 254) { length-=255; *op++ = 255; }
*op++ = (BYTE)length;
}
else *token += length;
// Test end of chunk
if (ip > mflimit) { anchor = ip; break; }
@ -713,7 +726,6 @@ int LZ4_uncompress(const char* source,
unsigned token;
size_t length;
size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};
#if LZ4_ARCH64
size_t dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
@ -723,13 +735,15 @@ int LZ4_uncompress(const char* source,
// Main Loop
while (1)
{
size_t length;
// get runlength
token = *ip++;
if ((length=(token>>ML_BITS)) == RUN_MASK) { size_t len; for (;(len=*ip++)==255;length+=255){} length += len; }
// copy literals
cpy = op+length;
if unlikely(cpy>oend-COPYLENGTH)
if (cpy>oend-COPYLENGTH)
{
if (cpy != oend) goto _output_error; // Error : not enough place for another match (min 4) + 5 literals
memcpy(op, ip, length);
@ -740,7 +754,7 @@ int LZ4_uncompress(const char* source,
// get offset
LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
if unlikely(ref < (BYTE* const)dest) goto _output_error; // Error : offset create reference outside destination buffer
if unlikely(ref < (BYTE* const)dest) goto _output_error; // Error : offset outside destination buffer
// get matchlength
if ((length=(token&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; }
@ -762,16 +776,17 @@ int LZ4_uncompress(const char* source,
op += STEPSIZE-4; ref -= dec64;
} else { LZ4_COPYSTEP(ref,op); }
cpy = op + length - (STEPSIZE-4);
if (cpy>oend-COPYLENGTH)
if unlikely(cpy>oend-(COPYLENGTH)-(STEPSIZE-4))
{
if (cpy > oend) goto _output_error; // Error : request to write beyond destination buffer
if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals
LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH));
while(op<cpy) *op++=*ref++;
op=cpy;
if (op == oend) goto _output_error; // Check EOF (should never happen, since last 5 bytes are supposed to be literals)
continue;
}
LZ4_SECURECOPY(ref, op, cpy);
LZ4_WILDCOPY(ref, op, cpy);
op=cpy; // correction
}
@ -805,22 +820,29 @@ int LZ4_uncompress_unknownOutputSize(
#endif
// Special case
if unlikely(ip==iend) goto _output_error; // A correctly formed null-compressed LZ4 must have at least one byte (token=0)
// Main Loop
while (ip<iend)
while (1)
{
unsigned token;
size_t length;
// get runlength
token = *ip++;
if ((length=(token>>ML_BITS)) == RUN_MASK) { int s=255; while ((ip<iend) && (s==255)) { s=*ip++; length += s; } }
if ((length=(token>>ML_BITS)) == RUN_MASK)
{
int s=255;
while (likely(ip<iend) && (s==255)) { s=*ip++; length += s; }
}
// copy literals
cpy = op+length;
if ((cpy>oend-COPYLENGTH) || (ip+length>iend-COPYLENGTH))
if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS)))
{
if (cpy > oend) goto _output_error; // Error : writes beyond output buffer
if (ip+length != iend) goto _output_error; // Error : LZ4 format requires to consume all input at this stage
if (ip+length != iend) goto _output_error; // Error : LZ4 format requires to consume all input at this stage (no match within the last 11 bytes, and at least 8 remaining input bytes for another match+literals)
memcpy(op, ip, length);
op += length;
break; // Necessarily EOF, due to parsing restrictions
@ -829,10 +851,19 @@ int LZ4_uncompress_unknownOutputSize(
// get offset
LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
if (ref < (BYTE* const)dest) goto _output_error; // Error : offset creates reference outside of destination buffer
if unlikely(ref < (BYTE* const)dest) goto _output_error; // Error : offset outside of destination buffer
// get matchlength
if ((length=(token&ML_MASK)) == ML_MASK) { while (ip<iend) { int s = *ip++; length +=s; if (s==255) continue; break; } }
if ((length=(token&ML_MASK)) == ML_MASK)
{
while likely(ip<iend-(LASTLITERALS+1)) // Error : a minimum input bytes must remain for LASTLITERALS + token
{
int s = *ip++;
length +=s;
if (s==255) continue;
break;
}
}
// copy repeated sequence
if unlikely(op-ref<STEPSIZE)
@ -851,16 +882,17 @@ int LZ4_uncompress_unknownOutputSize(
op += STEPSIZE-4; ref -= dec64;
} else { LZ4_COPYSTEP(ref,op); }
cpy = op + length - (STEPSIZE-4);
if (cpy>oend-COPYLENGTH)
if unlikely(cpy>oend-(COPYLENGTH+(STEPSIZE-4)))
{
if (cpy > oend) goto _output_error; // Error : request to write outside of destination buffer
if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals
LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH));
while(op<cpy) *op++=*ref++;
op=cpy;
if (op == oend) goto _output_error; // Check EOF (should never happen, since last 5 bytes are supposed to be literals)
continue;
}
LZ4_SECURECOPY(ref, op, cpy);
LZ4_WILDCOPY(ref, op, cpy);
op=cpy; // correction
}

53
lz4hc.c
View File

@ -377,41 +377,29 @@ forceinline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4,
const BYTE* ref;
INITBASE(base,hc4->base);
int nbAttempts=MAX_NB_ATTEMPTS;
size_t ml=0;
size_t repl=0, ml=0;
U16 delta;
// HC4 match finder
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
#if 1
#define REPEAT_OPTIMIZATION
#ifdef REPEAT_OPTIMIZATION
// Detect repetitive sequences of length <= 4
if (ref >= ip-4) // potential repetition
{
if (A32(ref) == A32(ip)) // confirmed
{
const U16 delta = (U16)(ip-ref);
const BYTE* ptr = ip;
const BYTE* end;
ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
end = ip + ml - (MINMATCH-1);
while(ptr < end-delta)
{
DELTANEXT(ptr) = delta; // Pre-Load
ptr++;
}
do
{
DELTANEXT(ptr) = delta;
HashTable[HASH_VALUE(ptr)] = (ptr) - base; // Head of chain
ptr++;
} while(ptr < end);
hc4->nextToUpdate = end;
delta = (U16)(ip-ref);
repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
*matchpos = ref;
}
ref = GETNEXT(ref);
}
#endif
while ((ref >= ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts))
while ((ref >= ip-MAX_DISTANCE) && (nbAttempts))
{
nbAttempts--;
if (*(ref+ml) == *(ip+ml))
@ -423,6 +411,29 @@ forceinline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4,
ref = GETNEXT(ref);
}
#ifdef REPEAT_OPTIMIZATION
// Complete table
if (repl)
{
const BYTE* ptr = ip;
const BYTE* end;
end = ip + repl - (MINMATCH-1);
while(ptr < end-delta)
{
DELTANEXT(ptr) = delta; // Pre-Load
ptr++;
}
do
{
DELTANEXT(ptr) = delta;
HashTable[HASH_VALUE(ptr)] = (ptr) - base; // Head of chain
ptr++;
} while(ptr < end);
hc4->nextToUpdate = end;
}
#endif
return (int)ml;
}
@ -440,7 +451,7 @@ forceinline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4,
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
while ((ref >= ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts))
while ((ref >= ip-MAX_DISTANCE) && (nbAttempts))
{
nbAttempts--;
if (*(startLimit + longest) == *(ref - delta + longest))