LZ4 HC : extended detection window. Thanks to Adrien Grand.
Fuzzer : more tests cases lz4demo : detect write errors. Thanks to Dima Tisnek bench.c : compatibility with Solaris 64. Thanks to Thorbjørn Willoch LZ4_compressBound() : now both in inline function and macro format. Thanks to Jacob Gorm Hansen git-svn-id: https://lz4.googlecode.com/svn/trunk@84 650e7d94-2a16-8b24-b05c-7c0b3f6821cd
This commit is contained in:
yann.collet.73@gmail.com
parent
43a03b41b2
commit
ffb27d4eca
2
bench.c
2
bench.c
@ -33,7 +33,7 @@
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
|
||||
# define _LARGEFILE_SOURCE
|
||||
# define FILE_OFFSET_BITS=64
|
||||
#else
|
||||
#elif ! defined(__LP64__) // No point defining Large file for 64 bit
|
||||
# define _LARGEFILE64_SOURCE
|
||||
#endif
|
||||
|
||||
|
||||
59
fuzzer.c
59
fuzzer.c
@ -33,7 +33,7 @@
|
||||
// Includes
|
||||
//**************************************
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h> // fgets
|
||||
#include <stdio.h> // fgets, sscanf
|
||||
#include <sys/timeb.h> // timeb
|
||||
#include "lz4.h"
|
||||
|
||||
@ -97,11 +97,12 @@ int main() {
|
||||
unsigned long long bytes = 0;
|
||||
unsigned long long cbytes = 0;
|
||||
unsigned char buf[LEN];
|
||||
# define FUZ_max LZ4_compressBound(LEN)
|
||||
unsigned char testOut[LEN+1];
|
||||
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
|
||||
# define FUZ_avail ROUND_PAGE(FUZ_max)
|
||||
const int off_full = FUZ_avail - FUZ_max;
|
||||
unsigned char cbuf[FUZ_avail + PAGE_SIZE];
|
||||
unsigned int seed, cur_seq, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();
|
||||
unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();
|
||||
int i, j, k, ret, len;
|
||||
char userInput[30] = {0};
|
||||
|
||||
@ -120,6 +121,7 @@ int main() {
|
||||
|
||||
for (i = 0; i < NB_ATTEMPTS; i++) {
|
||||
printf("\r%7i /%7i\r", i, NB_ATTEMPTS);
|
||||
|
||||
FUZ_rand(&seed);
|
||||
for (j = 0; j < NUM_SEQ; j++) {
|
||||
seeds[j] = FUZ_rand(&seed) << 8;
|
||||
@ -136,23 +138,64 @@ int main() {
|
||||
}
|
||||
buf[j] = FUZ_rand(&cur_seq) >> 16;
|
||||
}
|
||||
|
||||
// Test compression
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
|
||||
if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
len = ret;
|
||||
|
||||
// Test compression with output size being exactly what's necessary
|
||||
// Test decoding with output size being exactly what's necessary => must work
|
||||
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN);
|
||||
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with one byte missing => must fail
|
||||
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1);
|
||||
if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with one byte too much => must fail
|
||||
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1);
|
||||
if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with enough output size => must work
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);
|
||||
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with output size being exactly what's necessary => should work
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN);
|
||||
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with output size being one byte too short => must fail
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);
|
||||
if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with input size being one byte too short => must fail
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);
|
||||
if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test decoding with input size being one byte too large => must fail
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);
|
||||
if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test compression with output size being exactly what's necessary (should work)
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);
|
||||
if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); return 1; }
|
||||
if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); return 1; }
|
||||
if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; }
|
||||
if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }
|
||||
|
||||
// Test compression with just one missing byte into output buffer => must fail
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);
|
||||
if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); return 1; }
|
||||
if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); return 1; }
|
||||
if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; }
|
||||
if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; }
|
||||
|
||||
bytes += LEN;
|
||||
cbytes += len;
|
||||
}
|
||||
|
||||
printf("all tests completed successfully \n");
|
||||
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
|
||||
getchar();
|
||||
return 0;
|
||||
|
||||
_output_error:
|
||||
getchar();
|
||||
return 1;
|
||||
}
|
||||
|
||||
3
lz4.c
3
lz4.c
@ -91,7 +91,7 @@
|
||||
//**************************************
|
||||
// Compiler Options
|
||||
//**************************************
|
||||
#if __STDC_VERSION__ >= 199901L // C99
|
||||
#if __STDC_VERSION__ >= 199901L // C99
|
||||
/* "restrict" is a known keyword */
|
||||
#else
|
||||
# define restrict // Disable restrict
|
||||
@ -100,7 +100,6 @@
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
|
||||
#ifdef _MSC_VER // Visual Studio
|
||||
# define inline __forceinline // Visual is not C99, but supports some kind of inline
|
||||
# include <intrin.h> // For Visual 2005
|
||||
# if LZ4_ARCH64 // 64-bit
|
||||
# pragma intrinsic(_BitScanForward64) // For Visual 2005
|
||||
|
||||
15
lz4.h
15
lz4.h
@ -38,6 +38,14 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
//**************************************
|
||||
// Compiler Options
|
||||
//**************************************
|
||||
#ifdef _MSC_VER // Visual Studio
|
||||
# define inline __inline // Visual is not C99, but supports some kind of inline
|
||||
#endif
|
||||
|
||||
|
||||
//****************************
|
||||
// Simple Functions
|
||||
//****************************
|
||||
@ -50,7 +58,7 @@ LZ4_compress() :
|
||||
Compresses 'isize' bytes from 'source' into 'dest'.
|
||||
Destination buffer must be already allocated,
|
||||
and must be sized to handle worst cases situations (input data not compressible)
|
||||
Worst case size evaluation is provided by macro LZ4_compressBound()
|
||||
Worst case size evaluation is provided by function LZ4_compressBound()
|
||||
|
||||
isize : is the input size. Max supported value is ~1.9GB
|
||||
return : the number of bytes written in buffer dest
|
||||
@ -70,12 +78,15 @@ LZ4_uncompress() :
|
||||
// Advanced Functions
|
||||
//****************************
|
||||
|
||||
#define LZ4_compressBound(isize) (isize + (isize/255) + 16)
|
||||
static inline int LZ4_compressBound(int isize) { return ((isize) + ((isize)/255) + 16); }
|
||||
#define LZ4_COMPRESSBOUND( isize) ((isize) + ((isize)/255) + 16)
|
||||
|
||||
/*
|
||||
LZ4_compressBound() :
|
||||
Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
|
||||
primarily useful for memory allocation of output buffer.
|
||||
inline function is recommended for the general case,
|
||||
but macro is also provided when results need to be evaluated at compile time (such as table size allocation).
|
||||
|
||||
isize : is the input size. Max supported value is ~1.9GB
|
||||
return : maximum output size in a "worst case" scenario
|
||||
|
||||
19
lz4demo.c
19
lz4demo.c
@ -53,7 +53,7 @@
|
||||
|
||||
|
||||
//**************************************
|
||||
// Compiler functions
|
||||
// Compiler-specific functions
|
||||
//**************************************
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
|
||||
@ -91,9 +91,9 @@ static inline unsigned int swap32(unsigned int x) {
|
||||
// Architecture Macros
|
||||
//**************************************
|
||||
static const int one = 1;
|
||||
#define CPU_LITTLE_ENDIAN (*(char*)(&one))
|
||||
#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)
|
||||
#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); }
|
||||
#define CPU_LITTLE_ENDIAN (*(char*)(&one))
|
||||
#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)
|
||||
#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); }
|
||||
|
||||
|
||||
//**************************************
|
||||
@ -177,6 +177,7 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
|
||||
int r;
|
||||
int displayLevel = (compressionlevel>0);
|
||||
clock_t start, end;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
// Init
|
||||
@ -199,7 +200,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
|
||||
u32var = ARCHIVE_MAGICNUMBER;
|
||||
LITTLE_ENDIAN32(u32var);
|
||||
*(unsigned int*)out_buff = u32var;
|
||||
fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput);
|
||||
sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput);
|
||||
if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; }
|
||||
|
||||
// Main Loop
|
||||
while (1)
|
||||
@ -220,7 +222,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
|
||||
LITTLE_ENDIAN32(outSize);
|
||||
* (unsigned int*) out_buff = outSize;
|
||||
LITTLE_ENDIAN32(outSize);
|
||||
fwrite(out_buff, 1, outSize+4, foutput);
|
||||
sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
|
||||
if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; }
|
||||
}
|
||||
|
||||
// Status
|
||||
@ -254,6 +257,7 @@ int decode_file(char* input_filename, char* output_filename)
|
||||
FILE* foutput;
|
||||
clock_t start, end;
|
||||
int r;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
// Init
|
||||
@ -291,7 +295,8 @@ int decode_file(char* input_filename, char* output_filename)
|
||||
filesize += sinkint;
|
||||
|
||||
// Write Block
|
||||
fwrite(out_buff, 1, sinkint, foutput);
|
||||
sizeCheck = fwrite(out_buff, 1, sinkint, foutput);
|
||||
if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; }
|
||||
}
|
||||
|
||||
// Status
|
||||
|
||||
4
lz4hc.c
4
lz4hc.c
@ -337,7 +337,7 @@ inline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const
|
||||
// HC4 match finder
|
||||
LZ4HC_Insert(hc4, ip);
|
||||
ref = HASH_POINTER(ip);
|
||||
while ((ref > (ip-MAX_DISTANCE)) && (nbAttempts))
|
||||
while ((ref >= (ip-MAX_DISTANCE)) && (nbAttempts))
|
||||
{
|
||||
nbAttempts--;
|
||||
if (*(ref+ml) == *(ip+ml))
|
||||
@ -380,7 +380,7 @@ inline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const
|
||||
LZ4HC_Insert(hc4, ip);
|
||||
ref = HASH_POINTER(ip);
|
||||
|
||||
while ((ref > ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts))
|
||||
while ((ref >= ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts))
|
||||
{
|
||||
nbAttempts--;
|
||||
if (*(startLimit + longest) == *(ref - delta + longest))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user