AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #3 from Cyan4973/beta

Browse Source 
Beta
This commit is contained in:
Cyan4973 2014-04-23 00:45:28 +02:00
commit dac2a0946d
17 changed files with 204 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml Normal file
View File

@ -0,0 +1,6 @@
language: c
compiler: gcc
script: make test
before_install:
- sudo apt-get update -qq
- sudo apt-get install -qq gcc-multilib

2
LICENSE
View File

@ -1,5 +1,5 @@
LZ4 Library
Copyright (c) 2013, Yann Collet
Copyright (c) 2011-2014, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,

10
Makefile
View File

@ -30,7 +30,7 @@
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
export RELEASE=r116
export RELEASE=r117
LIBVER_MAJOR=1
LIBVER_MINOR=0
LIBVER_PATCH=0
@ -69,13 +69,14 @@ else
endif
TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \
lz4_format_description.txt Makefile NEWS LICENSE \
lz4_format_description.txt Makefile NEWS LICENSE README.md \
cmake_unofficial/CMakeLists.txt \
$(PRGDIR)/fullbench.c $(PRGDIR)/fuzzer.c $(PRGDIR)/lz4cli.c \
$(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \
$(PRGDIR)/bench.c $(PRGDIR)/bench.h \
$(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \
$(PRGDIR)/lz4.1 $(PRGDIR)/Makefile $(PRGDIR)/COPYING
$(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \
$(PRGDIR)/Makefile $(PRGDIR)/COPYING
NONTEXT = LZ4_Streaming_Format.odt
SOURCES = $(TEXT) $(NONTEXT)
@ -144,4 +145,7 @@ dist: clean
@sha1sum $(DISTRIBNAME) > $(DISTRIBNAME).sha1
@echo Distribution $(DISTRIBNAME) built
test: lz4programs
@cd $(PRGDIR); $(MAKE) -e $@
endif

6
NEWS
View File

@ -1,3 +1,9 @@
r117:
Added : man pages for lz4c and lz4cat
Added : automated tests on Travis, thanks to Takayuki Matsuoka !
fix : block-dependency command line (issue 127)
fix : lz4fullbench (issue 128)
r116:
hotfix (issue 124 & 125)

49
README.md Normal file
View File

@ -0,0 +1,49 @@
LZ4 - Extremely fast compression
================================
LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio.
This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/).
The intention is to offer github's capabilities to lz4 users, such as cloning, branch, or source download.
The "master" branch will reflect, the status of lz4 at its official homepage. Other branches will also exist, typically to fix some open issues or new requirements, and be available for testing before merge into master.
Benchmarks
-------------------------
The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.2)](http://encode.ru/threads/1371-Filesystem-benchmark?p=33548&viewfull=1#post33548) compiled with GCC v4.6.1 on Linux Ubuntu 64-bits v11.10,
The reference system uses a Core i5-3340M @2.7GHz.
Benchmark evaluates the compression of reference [Silesia Corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia) in single-thread mode.
<table>
<tr>
<th>Compressor</th><th>Ratio</th><th>Compression</th><th>Decompression</th>
</tr>
<tr>
<th>LZ4 (r101)</th><th>2.084</th><th>422 MB/s</th><th>1820 MB/s</th>
</tr>
<tr>
<th>LZO 2.06</th><th>2.106</th><th>414 MB/s</th><th>600 MB/s</th>
</tr>
<tr>
<th>QuickLZ 1.5.1b6</th><th>2.237</th><th>373 MB/s</th><th>420 MB/s</th>
</tr>
<tr>
<th>Snappy 1.1.0</th><th>2.091</th><th>323 MB/s</th><th>1070 MB/s</th>
</tr>
<tr>
<th>LZF</th><th>2.077</th><th>270 MB/s</th><th>570 MB/s</th>
</tr>
<tr>
<th>zlib 1.2.8 -1</th><th>2.730</th><th>65 MB/s</th><th>280 MB/s</th>
</tr>
<tr>
<th>LZ4 HC (r101)</th><th>2.720</th><th>25 MB/s</th><th>2080 MB/s</th>
</tr>
<tr>
<th>zlib 1.2.8 -6</th><th>3.099</th><th>21 MB/s</th><th>300 MB/s</th>
</tr>
</table>

2
lz4.h
View File

@ -1,7 +1,7 @@
/*
LZ4 - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2013, Yann Collet.
Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without

2
lz4hc.h
View File

@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
Copyright (C) 2011-2013, Yann Collet.
Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without

25
programs/Makefile
View File

@ -41,6 +41,7 @@ BINDIR=$(PREFIX)/bin
MANDIR=$(PREFIX)/share/man/man1
LZ4DIR=..
TEST_FILES = COPYING
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@ -85,11 +86,15 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin))
install: lz4 lz4c
@echo Installing binaries
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4
@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
@install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c
@echo Installing man pages
@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
@install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1
@install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1
@echo lz4 installation completed
uninstall:
@ -99,4 +104,24 @@ uninstall:
[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
@echo lz4 successfully uninstalled
test: test-lz4 test-lz4c test-lz4c32 test-fuzzer test-fuzzer32 test-fullbench test-fullbench32
test-lz4:
test-lz4c:
test-lz4c32:
test-fuzzer: fuzzer
./fuzzer --no-prompt
test-fuzzer32: fuzzer32
./fuzzer32 --no-prompt
test-fullbench: fullbench
./fullbench --no-prompt $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(TEST_FILES)
endif

2
programs/bench.c
View File

@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2013
Copyright (C) Yann Collet 2012-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify

2
programs/bench.h
View File

@ -1,6 +1,6 @@
/*
bench.h - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2013
Copyright (C) Yann Collet 2012-2014
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

24
programs/fullbench.c
View File

@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2013
Copyright (C) Yann Collet 2012-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify
@ -50,6 +50,7 @@
#include <stdio.h> // fprintf, fopen, ftello64
#include <sys/types.h> // stat64
#include <sys/stat.h> // stat64
#include <string.h> // strcmp
// Use ftime() if gettimeofday() is not available on your target
#if defined(BMK_LEGACY_TIMER)
@ -138,6 +139,7 @@ struct chunkParameters
// MACRO
//**************************************
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__)
@ -151,6 +153,7 @@ static int compressionTest = 1;
static int decompressionTest = 1;
static int compressionAlgo = ALL_COMPRESSORS;
static int decompressionAlgo = ALL_DECOMPRESSORS;
static int no_prompt = 0;
void BMK_SetBlocksize(int bsize)
{
@ -281,12 +284,12 @@ static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char
static void* stateLZ4HC;
static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compress_withState(stateLZ4HC, in, out, inSize);
return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
}
static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
}
static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
@ -478,7 +481,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
double averageTime;
int milliTime;
DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
PROGRESS("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
nb_loops = 0;
@ -502,7 +505,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
if (averageTime < bestTime) bestTime = averageTime;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
PROGRESS("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
}
if (ratio<100.)
@ -547,7 +550,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
int milliTime;
U32 crcDecoded;
DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
PROGRESS("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
nb_loops = 0;
milliTime = BMK_GetMilliStart();
@ -567,7 +570,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
averageTime = (double)milliTime / nb_loops;
if (averageTime < bestTime) bestTime = averageTime;
DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
PROGRESS("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
// CRC Checking
crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
@ -627,7 +630,7 @@ int usage_advanced()
{
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR);
DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
DISPLAY( " -d# : test only decompression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
//DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
@ -658,6 +661,11 @@ int main(int argc, char** argv)
char* argument = argv[i];
if(!argument) continue; // Protection if argument empty
if (!strcmp(argument, "--no-prompt"))
{
no_prompt = 1;
continue;
}
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')

25
programs/fuzzer.c
View File

@ -34,6 +34,7 @@
#include <stdlib.h>
#include <stdio.h> // fgets, sscanf
#include <sys/timeb.h> // timeb
#include <string.h> // strcmp
#include "lz4.h"
#include "lz4hc.h"
@ -121,8 +122,8 @@ int FUZ_SecurityTest()
}
//int main(int argc, char *argv[]) {
int main() {
int main(int argc, char** argv) {
const int no_prompt = (argc > 1) && (!strcmp(argv[1], "--no-prompt"));
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
unsigned long long hcbytes = 0;
@ -136,14 +137,14 @@ int main() {
int i, j, k, ret, len, lenHC, attemptNb;
char userInput[30] = {0};
# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }
# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb);
# define FUZ_DISPLAYTEST testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);
printf("Select an Initialisation number (default : random) : ");
fflush(stdout);
if ( fgets(userInput, sizeof userInput, stdin) )
if ( no_prompt || fgets(userInput, sizeof userInput, stdin) )
{
if ( sscanf(userInput, "%u", &seed) == 1 ) {}
else seed = FUZ_GetMilliSpan(timestamp);
@ -160,7 +161,17 @@ int main() {
{
int testNb = 0;
printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);
// note : promptThrottle is throtting stdout to prevent
// Travis-CI's output limit (10MB) and false hangup detection.
const int promptThrottle = (attemptNb % (NB_ATTEMPTS / 100) == 0);
if (!no_prompt || attemptNb == 0 || promptThrottle)
{
printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);
if (no_prompt)
{
fflush(stdout);
}
}
for (j = 0; j < NUM_SEQ; j++) {
seeds[j] = FUZ_rand(&randState) << 8;
@ -293,10 +304,10 @@ int main() {
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
getchar();
if(!no_prompt) getchar();
return 0;
_output_error:
getchar();
if(!no_prompt) getchar();
return 1;
}

33
programs/lz4c.1 Normal file
View File

@ -0,0 +1,33 @@
\"
\" lz4c.1: This is a manual page for 'lz4c' program. This file is part of the
\" lz4 <https://code.google.com/p/lz4/> project.
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH lz4c "1" "2014-04-15" "lz4c" "User Commands"
.SH NAME
\fBlz4\fR - Extremely fast compression algorithm
.SH SYNOPSIS
.TP 5
\fBlz4c\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
.SH DESCRIPTION
.PP
\fBlz4c\fR is the legacy version of \fBlz4\fR.
As such, it supports older supplementary legacy commands.
\fBlz4c\fR is now deprecated.
It is recommended to use \fBlz4\fR instead whenever possible.
To get a list of commands specific to lz4c, do :
lz4c -h
.SH BUGS
Report bugs at:- https://code.google.com/p/lz4/
.SH AUTHOR
Yann Collet

34
programs/lz4cat.1 Normal file
View File

@ -0,0 +1,34 @@
\"
\" lz4cat.1: This is a manual page for 'lz4cat' program. This file is part of
\" the lz4 <https://code.google.com/p/lz4/> project.
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH lz4cat "1" "2014-04-15" "lz4cat" "User Commands"
.SH NAME
\fBlz4cat\fR - Extremely fast compression algorithm
.SH SYNOPSIS
.TP 5
\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
.SH DESCRIPTION
.PP
\fBlz4\fR is an extremely fast lossless compression algorithm.
\fBlz4cat\fR is an utility based on \fBlz4\fR.
\fBlz4cat\fR is equivalent to \fBlz4 -cd\fR,
which forces decompression and redirect its output to the console.
All other options are the same as \fBlz4\fR ones (man lz4).
.SH BUGS
Report bugs at:- https://code.google.com/p/lz4/
.SH AUTHOR
Yann Collet

6
programs/lz4cli.c
View File

@ -1,6 +1,6 @@
/*
LZ4cli.c - LZ4 Command Line Interface
Copyright (C) Yann Collet 2011-2013
Copyright (C) Yann Collet 2011-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify
@ -407,11 +407,11 @@ int main(int argc, char** argv)
int B = argument[1] - '0';
int S = 1 << (8 + 2*B);
BMK_SetBlocksize(S);
LZ4IO_setBlockSizeID(B);
blockSize = LZ4IO_setBlockSizeID(B);
argument++;
break;
}
case 'D': LZ4IO_setBlockMode(independentBlocks); argument++; break;
case 'D': LZ4IO_setBlockMode(chainedBlocks); argument++; break;
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break;
default : exitBlockProperties=1;
}

2
programs/lz4io.c
View File

@ -1,6 +1,6 @@
/*
LZ4io.c - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2013
Copyright (C) Yann Collet 2011-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify

2
programs/lz4io.h
View File

@ -1,6 +1,6 @@
/*
LZ4io.h - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2013
Copyright (C) Yann Collet 2011-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify