AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-11-22 10:38:44 +01:00
commit 2cbd0233b8
28 changed files with 408 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
.travis.yml
View File

@ -17,10 +17,35 @@ matrix:
- os: linux
sudo: false
env: Ubu=12.04cont Cmd="make gpptest-native && make clean && make examples && make clean && make cmake && make clean && make travis-install && make clean && make clangtest-native" COMPILER=cc
env: Ubu=12.04cont Cmd="make gpptest && make clean examples && make clean cmake && make clean travis-install && make clean clangtest" COMPILER=cc
# 14.04 LTS Server Edition 64 bit
- env: Ubu=14.04 Cmd='make -C tests test MOREFLAGS=-mx32' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu=14.04 Cmd='make usan' COMPILER=clang
dist: trusty
sudo: required
addons:
apt:
packages:
- clang
- env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
- env: Ubu=14.04 Cmd='make -C tests test-lz4c32 test-fullbench32 versionsTest' COMPILER=cc
dist: trusty
sudo: required
@ -40,21 +65,15 @@ matrix:
- libc6-dev-i386
- gcc-multilib
- env: Ubu=14.04 Cmd='make sanitize' COMPILER=clang
- env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror' COMPILER=gcc-6
dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- valgrind
- env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang
dist: trusty
sudo: required
addons:
apt:
packages:
- clang
- gcc-6
- env: Ubu=14.04 Cmd='make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static' COMPILER=arm-linux-gnueabi-gcc
dist: trusty
@ -69,25 +88,7 @@ matrix:
- gcc-aarch64-linux-gnu
- libc6-dev-arm64-cross
- env: Ubu=14.04 Cmd='make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64' COMPILER=powerpc-linux-gnu-gcc
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
- env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror' COMPILER=gcc-5
- env: Ubu=14.04 Cmd='make -C tests test-lz4 clean test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror' COMPILER=gcc-5
dist: trusty
sudo: required
addons:
@ -100,16 +101,6 @@ matrix:
- gcc-5
- gcc-5-multilib
- env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror' COMPILER=gcc-6
dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- env: Ubu=14.04 Cmd='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8
dist: trusty
sudo: required
@ -121,6 +112,36 @@ matrix:
packages:
- clang-3.8
- env: Ubu=14.04 Cmd='make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64' COMPILER=powerpc-linux-gnu-gcc
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang
dist: trusty
sudo: required
addons:
apt:
packages:
- clang
- env: Ubu=14.04 Cmd='make clean all CC=gcc-4.4 MOREFLAGS=-Werror' COMPILER=gcc-4.4
dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- libc6-dev-i386
- gcc-multilib
- gcc-4.4
script:
- echo Cmd=$Cmd
- $COMPILER -v

39
Makefile
View File

@ -1,6 +1,6 @@
# ################################################################
# LZ4 - Makefile
# Copyright (C) Yann Collet 2011-2015
# Copyright (C) Yann Collet 2011-2016
# All rights reserved.
#
# BSD license
@ -36,9 +36,10 @@ VOID := /dev/null
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
PRGDIR = programs
LZ4DIR = lib
PRGDIR = programs
TESTDIR = tests
EXDIR = examples
# Define nul output
@ -51,14 +52,22 @@ endif
.PHONY: default all lib lz4 clean test versionsTest examples
default: lz4
default: lib lz4-release
all: lib lz4
all:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
@$(MAKE) -C $(TESTDIR) $@
@$(MAKE) -C $(EXDIR) $@
lib:
@$(MAKE) -C $(LZ4DIR) all
@$(MAKE) -C $(LZ4DIR)
lz4:
@$(MAKE) -C $(PRGDIR) $@
@cp $(PRGDIR)/lz4$(EXT) .
lz4-release:
@$(MAKE) -C $(PRGDIR)
@cp $(PRGDIR)/lz4$(EXT) .
@ -66,6 +75,7 @@ clean:
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(EXDIR) $@ > $(VOID)
@$(MAKE) -C examples $@ > $(VOID)
@$(RM) lz4$(EXT)
@echo Cleaning completed
@ -103,8 +113,11 @@ clangtest-native: clean
@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) native CC=clang
@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
sanitize: clean
CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1
usan: clean
CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
usan32: clean
CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
staticAnalyze: clean
CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
@ -113,8 +126,8 @@ platformTest: clean
@echo "\n ---- test lz4 with $(CC) compiler ----"
@$(CC) -v
CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(PRGDIR) native
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) native
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(PRGDIR) all
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all
$(MAKE) -C $(TESTDIR) test-platform
versionsTest: clean
@ -148,11 +161,11 @@ gpptest: clean
CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
CC=g++ $(MAKE) -C $(TESTDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
gpptest-native: clean
gpptest32: clean
g++ -v
CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
CC=g++ $(MAKE) -C $(PRGDIR) native CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
CC=g++ $(MAKE) -C $(PRGDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
c_standards: clean
$(MAKE) all MOREFLAGS="-std=gnu90 -Werror"

5
NEWS
View File

@ -1,3 +1,8 @@
v1.7.4
Improved : much better speed in -mx32 mode
cli : fix : Large file support in 32-bits mode on Mac OS-X
fix : compilation on gcc 4.4 (#282), reported by Antoine Martin
v1.7.3
Changed : moved to versioning; package, cli and library have same version number
Improved: Small decompression speed boost

46
README.md
View File

@ -43,29 +43,35 @@ LZ4 library is provided as open-source software using BSD 2-Clause license.
Benchmarks
-------------------------
The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.3)]
compiled with GCC v4.8.2 on Linux Mint 64-bits v17.
The reference system uses a Core i5-4300U @1.9GHz.
The benchmark uses [lzbench], from @inikep
compiled with GCC v6.2.0 on Linux 64-bits.
The reference system uses a Core i7-3930K CPU @ 4.5GHz.
Benchmark evaluates the compression of reference [Silesia Corpus]
in single-thread mode.
| Compressor | Ratio | Compression | Decompression |
| ---------- | ----- | ----------- | ------------- |
| memcpy | 1.000 | 4200 MB/s | 4200 MB/s |
|**LZ4 fast 17 (r129)**| 1.607 |**690 MB/s** | **2220 MB/s** |
|**LZ4 default (r129)**|**2.101**|**385 MB/s** | **1850 MB/s** |
| LZO 2.06 | 2.108 | 350 MB/s | 510 MB/s |
| QuickLZ 1.5.1.b6 | 2.238 | 320 MB/s | 380 MB/s |
| Snappy 1.1.0 | 2.091 | 250 MB/s | 960 MB/s |
| [Zstandard] 0.5.1 | 2.876 | 240 MB/s | 620 MB/s |
| LZF v3.6 | 2.073 | 175 MB/s | 500 MB/s |
| [zlib] 1.2.8 -1 | 2.730 | 59 MB/s | 250 MB/s |
|**LZ4 HC (r129)** |**2.720**| 22 MB/s | **1830 MB/s** |
| [zlib] 1.2.8 -6 | 3.099 | 18 MB/s | 270 MB/s |
[lzbench]: https://github.com/inikep/lzbench
[Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
| Compressor | Ratio | Compression | Decompression |
| ---------- | ----- | ----------- | ------------- |
| memcpy | 1.000 | 7300 MB/s | 7300 MB/s |
|**LZ4 fast 8 (v1.7.3)**| 1.799 |**911 MB/s** | **3360 MB/s** |
|**LZ4 default (v1.7.3)**|**2.101**|**625 MB/s** | **3220 MB/s** |
| LZO 2.09 | 2.108 | 620 MB/s | 845 MB/s |
| QuickLZ 1.5.0 | 2.238 | 510 MB/s | 600 MB/s |
| Snappy 1.1.3 | 2.091 | 450 MB/s | 1550 MB/s |
| LZF v3.6 | 2.073 | 365 MB/s | 820 MB/s |
| [Zstandard] 1.1.1 -1 | 2.876 | 330 MB/s | 930 MB/s |
| [Zstandard] 1.1.1 -3 | 3.164 | 200 MB/s | 810 MB/s |
| [zlib] deflate 1.2.8 -1| 2.730 | 100 MB/s | 370 MB/s |
|**LZ4 HC -9 (v1.7.3)** |**2.720**| 34 MB/s | **3240 MB/s** |
| [zlib] deflate 1.2.8 -6| 3.099 | 33 MB/s | 390 MB/s |
[zlib]: http://www.zlib.net/
[Zstandard]: http://www.zstd.net/
LZ4 is also compatible and well optimized for x32 mode, for which it provides +10% speed performance.
Documentation
-------------------------
@ -77,6 +83,9 @@ Organizing these blocks and providing a common header format to handle their con
is the purpose of the Frame format, defined into [lz4_Frame_format].
Interoperable versions of LZ4 must respect this frame format.
[lz4_Block_format]: doc/lz4_Block_format.md
[lz4_Frame_format]: doc/lz4_Frame_format.md
Other source versions
-------------------------
@ -86,9 +95,4 @@ many contributors have created versions of lz4 in multiple languages
(Java, C#, Python, Perl, Ruby, etc.).
A list of known source ports is maintained on the [LZ4 Homepage].
[Open-Source Benchmark program by m^2 (v0.14.3)]: http://encode.ru/threads/1371-Filesystem-benchmark?p=34029&viewfull=1#post34029
[Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
[lz4_Block_format]: doc/lz4_Block_format.md
[lz4_Frame_format]: doc/lz4_Frame_format.md
[LZ4 Homepage]: http://www.lz4.org

54
examples/HCStreaming_ringBuffer.c
View File

@ -6,7 +6,7 @@
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
# define _CRT_SECURE_NO_WARNINGS /* for MSVC */
# define snprintf sprintf_s
#endif
@ -59,17 +59,16 @@ void test_compress(FILE* outFp, FILE* inpFp)
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
for(;;)
{
for(;;) {
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
{
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
#define CMPBUFSIZE (LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES))
{ char cmpBuf[CMPBUFSIZE];
const int cmpBytes = LZ4_compress_HC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes, CMPBUFSIZE);
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
@ -94,13 +93,11 @@ void test_decompress(FILE* outFp, FILE* inpFp)
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
for(;;)
{
for(;;) {
int cmpBytes = 0;
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
char cmpBuf[CMPBUFSIZE];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
{ const size_t r0 = read_int32(inpFp, &cmpBytes);
size_t r1;
if(r0 != 1 || cmpBytes <= 0)
break;
@ -110,8 +107,7 @@ void test_decompress(FILE* outFp, FILE* inpFp)
break;
}
{
char* const decPtr = &decBuf[decOffset];
{ char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0)
@ -135,8 +131,7 @@ size_t compare(FILE* f0, FILE* f1)
{
size_t result = 1;
for (;;)
{
for (;;) {
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
@ -144,16 +139,14 @@ size_t compare(FILE* f0, FILE* f1)
if ((r0==0) && (r1==0)) return 0; // success
if (r0 != r1)
{
if (r0 != r1) {
size_t smallest = r0;
if (r1<r0) smallest = r1;
result += smallest;
break;
}
if (memcmp(b0, b1, r0))
{
if (memcmp(b0, b1, r0)) {
unsigned errorPos = 0;
while ((errorPos < r0) && (b0[errorPos]==b1[errorPos])) errorPos++;
result += errorPos;
@ -167,7 +160,7 @@ size_t compare(FILE* f0, FILE* f1)
}
int main(int argc, char** argv)
int main(int argc, const char** argv)
{
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
@ -192,9 +185,8 @@ int main(int argc, char** argv)
printf("decoded = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
test_compress(outFp, inpFp);
@ -203,9 +195,8 @@ int main(int argc, char** argv)
}
// decompress
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
@ -214,9 +205,8 @@ int main(int argc, char** argv)
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
const size_t cmp = compare(inpFp, decFp);
if(0 == cmp) {
@ -229,10 +219,10 @@ int main(int argc, char** argv)
fclose(inpFp);
}
if (pause)
{
if (pause) {
int unused;
printf("Press enter to continue ...\n");
getchar();
unused = getchar(); (void)unused; /* silence static analyzer */
}
return 0;

7
examples/blockStreaming_ringBuffer.c
View File

@ -64,8 +64,9 @@ void test_compress(FILE* outFp, FILE* inpFp)
if (0 == inpBytes) break;
{
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = LZ4_compress_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
#define CMPBUFSIZE (LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES))
char cmpBuf[CMPBUFSIZE];
const int cmpBytes = LZ4_compress_fast_continue(lz4Stream, inpPtr, cmpBuf, inpBytes, CMPBUFSIZE, 0);
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
@ -90,7 +91,7 @@ void test_decompress(FILE* outFp, FILE* inpFp)
for(;;) {
int cmpBytes = 0;
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
char cmpBuf[CMPBUFSIZE];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);

43
examples/frameCompress.c
View File

@ -116,6 +116,7 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_o
static size_t get_block_size(const LZ4F_frameInfo_t* info) {
switch (info->blockSizeID) {
case LZ4F_default:
case LZ4F_max64KB: return 1 << 16;
case LZ4F_max256KB: return 1 << 18;
case LZ4F_max1MB: return 1 << 20;
@ -127,17 +128,14 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
}
static size_t decompress_file(FILE *in, FILE *out) {
void * const src = malloc(BUF_SIZE);
void *dst = NULL;
void *srcPtr = src;
void *srcEnd = src;
size_t srcSize = 0;
size_t dstSize = 0;
void* const src = malloc(BUF_SIZE);
void* dst = NULL;
size_t dstCapacity = 0;
LZ4F_dctx *dctx = NULL;
size_t ret;
/* Initialization */
if (!src) { perror("decompress_file(src)"); goto cleanup; }
ret = LZ4F_createDecompressionContext(&dctx, 100);
if (LZ4F_isError(ret)) {
printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
@ -147,15 +145,14 @@ static size_t decompress_file(FILE *in, FILE *out) {
/* Decompression */
ret = 1;
while (ret != 0) {
/* INVARIANT: At this point srcPtr == srcEnd */
/* Load more input */
srcSize = fread(src, 1, BUF_SIZE, in);
size_t srcSize = fread(src, 1, BUF_SIZE, in);
void* srcPtr = src;
void* srcEnd = srcPtr + srcSize;
if (srcSize == 0 || ferror(in)) {
printf("Decompress: not enough input or error reading file\n");
goto cleanup;
}
srcPtr = src;
srcEnd = srcPtr + srcSize;
/* Allocate destination buffer if it isn't already */
if (!dst) {
LZ4F_frameInfo_t info;
@ -169,6 +166,7 @@ static size_t decompress_file(FILE *in, FILE *out) {
*/
dstCapacity = get_block_size(&info);
dst = malloc(dstCapacity);
if (!dst) { perror("decompress_file(dst)"); goto cleanup; }
srcPtr += srcSize;
srcSize = srcEnd - srcPtr;
}
@ -179,7 +177,7 @@ static size_t decompress_file(FILE *in, FILE *out) {
*/
while (srcPtr != srcEnd && ret != 0) {
/* INVARIANT: Any data left in dst has already been written */
dstSize = dstCapacity;
size_t dstSize = dstCapacity;
ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
if (LZ4F_isError(ret)) {
printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
@ -191,7 +189,6 @@ static size_t decompress_file(FILE *in, FILE *out) {
printf("Writing %zu bytes\n", dstSize);
if (written != dstSize) {
printf("Decompress: Failed to write to file\n");
ret = 1;
goto cleanup;
}
}
@ -212,9 +209,8 @@ static size_t decompress_file(FILE *in, FILE *out) {
cleanup:
free(src);
if (dst) free(dst);
if (dctx) ret = LZ4F_freeDecompressionContext(dctx);
return ret;
free(dst);
return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
}
int compare(FILE* fp0, FILE* fp1)
@ -240,7 +236,7 @@ int compare(FILE* fp0, FILE* fp1)
return result;
}
int main(int argc, char **argv) {
int main(int argc, const char **argv) {
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
@ -259,9 +255,8 @@ int main(int argc, char **argv) {
printf("dec = [%s]\n", decFilename);
/* compress */
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
size_t sizeIn = 0;
size_t sizeOut = 0;
size_t ret;
@ -282,9 +277,8 @@ int main(int argc, char **argv) {
}
/* decompress */
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
size_t ret;
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
@ -300,9 +294,8 @@ int main(int argc, char **argv) {
}
/* verify */
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp);

2
lib/LICENSE
View File

@ -1,5 +1,5 @@
LZ4 Library
Copyright (c) 2011-2015, Yann Collet
Copyright (c) 2011-2016, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,

14
lib/Makefile
View File

@ -1,6 +1,6 @@
# ################################################################
# LZ4 library - Makefile
# Copyright (C) Yann Collet 2011-2015
# Copyright (C) Yann Collet 2011-2016
# All rights reserved.
#
# BSD license
@ -46,7 +46,7 @@ DESTDIR?=
PREFIX ?= /usr/local
CPPFLAGS= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
@ -70,12 +70,18 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
default: lib
default: lib-release
all: lib
all32: CFLAGS+=-m32
all32: all
lib: liblz4.a liblz4
lib-release: CFLAGS := -O3
lib-release: lib
liblz4.a: *.c
ifeq ($(BUILD_STATIC),yes)
@echo compiling static library
@ -96,7 +102,7 @@ else
endif
clean:
@$(RM) -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* liblz4.pc dll/liblz4.dll dll/liblz4.lib
@$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib
@echo Cleaning library completed

92
lib/lz4.c
View File

@ -141,22 +141,25 @@
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef uintptr_t uptrval;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef size_t uptrval; /* generally true, except OpenVMS-64 */
#endif
#if defined(__x86_64__)
typedef U64 reg_t; /* 64-bits in x32 mode */
#else
typedef size_t reg_t; /* 32-bits in x32 mode */
#endif
/*-************************************
* Reading and writing into memory
**************************************/
#define STEPSIZE sizeof(size_t)
static unsigned LZ4_64bits(void) { return sizeof(void*)==8; }
static unsigned LZ4_isLittleEndian(void)
{
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
@ -169,7 +172,7 @@ static unsigned LZ4_isLittleEndian(void)
static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; }
static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; }
static size_t LZ4_read_ARCH(const void* memPtr) { return *(const size_t*) memPtr; }
static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; }
static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
@ -178,11 +181,11 @@ static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
typedef union { U16 u16; U32 u32; size_t uArch; } __attribute__((packed)) unalign;
typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;
static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
static size_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
@ -199,9 +202,9 @@ static U32 LZ4_read32(const void* memPtr)
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
static size_t LZ4_read_ARCH(const void* memPtr)
static reg_t LZ4_read_ARCH(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
reg_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
static void LZ4_write16(void* memPtr, U16 value)
@ -250,12 +253,6 @@ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
const BYTE* s = (const BYTE*)srcPtr;
BYTE* const e = (BYTE*)dstEnd;
#if 0
const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1));
LZ4_copy8(d,s); if (d>e-9) return;
d+=l2; s+=l2;
#endif /* join to align */
do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e);
}
@ -292,10 +289,10 @@ static const int LZ4_minLength = (MFLIMIT+1);
/*-************************************
* Common functions
**************************************/
static unsigned LZ4_NbCommonBytes (register size_t val)
static unsigned LZ4_NbCommonBytes (register reg_t val)
{
if (LZ4_isLittleEndian()) {
if (LZ4_64bits()) {
if (sizeof(val)==8) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
@ -319,7 +316,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
# endif
}
} else /* Big Endian CPU */ {
if (LZ4_64bits()) {
if (sizeof(val)==8) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
@ -350,18 +347,19 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
}
}
#define STEPSIZE sizeof(reg_t)
static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while (likely(pIn<pInLimit-(STEPSIZE-1))) {
size_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; }
pIn += LZ4_NbCommonBytes(diff);
return (unsigned)(pIn - pStart);
}
if (LZ4_64bits()) if ((pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (unsigned)(pIn - pStart);
@ -419,9 +417,9 @@ static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
FORCE_INLINE U32 LZ4_hashPosition(const void* p, tableType_t const tableType)
FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
{
if ((LZ4_64bits()) && (tableType == byU32)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
return LZ4_hash4(LZ4_read32(p), tableType);
}
@ -458,7 +456,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy
/** LZ4_compress_generic() :
inlined, to ensure branches are decided at compilation time */
FORCE_INLINE int LZ4_compress_generic(
LZ4_stream_t_internal* const dictPtr,
LZ4_stream_t_internal* const cctx,
const char* const source,
char* const dest,
const int inputSize,
@ -472,10 +470,10 @@ FORCE_INLINE int LZ4_compress_generic(
const BYTE* ip = (const BYTE*) source;
const BYTE* base;
const BYTE* lowLimit;
const BYTE* const lowRefLimit = ip - dictPtr->dictSize;
const BYTE* const dictionary = dictPtr->dictionary;
const BYTE* const dictEnd = dictionary + dictPtr->dictSize;
const size_t dictDelta = dictEnd - (const BYTE*)source;
const BYTE* const lowRefLimit = ip - cctx->dictSize;
const BYTE* const dictionary = cctx->dictionary;
const BYTE* const dictEnd = dictionary + cctx->dictSize;
const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
const BYTE* anchor = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
@ -485,7 +483,6 @@ FORCE_INLINE int LZ4_compress_generic(
BYTE* const olimit = op + maxOutputSize;
U32 forwardH;
size_t refDelta=0;
/* Init conditions */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
@ -497,11 +494,11 @@ FORCE_INLINE int LZ4_compress_generic(
lowLimit = (const BYTE*)source;
break;
case withPrefix64k:
base = (const BYTE*)source - dictPtr->currentOffset;
lowLimit = (const BYTE*)source - dictPtr->dictSize;
base = (const BYTE*)source - cctx->currentOffset;
lowLimit = (const BYTE*)source - cctx->dictSize;
break;
case usingExtDict:
base = (const BYTE*)source - dictPtr->currentOffset;
base = (const BYTE*)source - cctx->currentOffset;
lowLimit = (const BYTE*)source;
break;
}
@ -509,11 +506,12 @@ FORCE_INLINE int LZ4_compress_generic(
if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
LZ4_putPosition(ip, dictPtr->hashTable, tableType, base);
LZ4_putPosition(ip, cctx->hashTable, tableType, base);
ip++; forwardH = LZ4_hashPosition(ip, tableType);
/* Main Loop */
for ( ; ; ) {
ptrdiff_t refDelta = 0;
const BYTE* match;
BYTE* token;
@ -529,7 +527,7 @@ FORCE_INLINE int LZ4_compress_generic(
if (unlikely(forwardIp > mflimit)) goto _last_literals;
match = LZ4_getPositionOnHash(h, dictPtr->hashTable, tableType, base);
match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
if (dict==usingExtDict) {
if (match < (const BYTE*)source) {
refDelta = dictDelta;
@ -539,7 +537,7 @@ FORCE_INLINE int LZ4_compress_generic(
lowLimit = (const BYTE*)source;
} }
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, dictPtr->hashTable, tableType, base);
LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
} while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
|| ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
@ -612,10 +610,10 @@ _next_match:
if (ip > mflimit) break;
/* Fill table */
LZ4_putPosition(ip-2, dictPtr->hashTable, tableType, base);
LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
/* Test next position */
match = LZ4_getPosition(ip, dictPtr->hashTable, tableType, base);
match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
if (dict==usingExtDict) {
if (match < (const BYTE*)source) {
refDelta = dictDelta;
@ -624,7 +622,7 @@ _next_match:
refDelta = 0;
lowLimit = (const BYTE*)source;
} }
LZ4_putPosition(ip, dictPtr->hashTable, tableType, base);
LZ4_putPosition(ip, cctx->hashTable, tableType, base);
if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
&& (match+MAX_DISTANCE>=ip)
&& (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
@ -667,12 +665,12 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
} else {
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
}
@ -711,7 +709,7 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
@ -885,7 +883,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
if (*srcSizePtr < LZ4_64Klimit)
return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
else
return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr);
return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr);
}
}
@ -933,7 +931,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
}
#define HASH_UNIT sizeof(size_t)
#define HASH_UNIT sizeof(reg_t)
int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
{
LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse;
@ -969,7 +967,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
{
if ((LZ4_dict->currentOffset > 0x80000000) ||
((size_t)LZ4_dict->currentOffset > (size_t)src)) { /* address space overflow */
((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */
/* rescale hash table */
U32 const delta = LZ4_dict->currentOffset - 64 KB;
const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
@ -1139,8 +1137,8 @@ FORCE_INLINE int LZ4_decompress_generic(
s = *ip++;
length += s;
} while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) );
if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
}
/* copy literals */
@ -1178,7 +1176,7 @@ FORCE_INLINE int LZ4_decompress_generic(
if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
length += s;
} while (s==255);
if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;
@ -1430,7 +1428,7 @@ static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base)
int LZ4_resetStreamState(void* state, char* inputBuffer)
{
if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer);
return 0;
}

9
lib/lz4.h
View File

@ -85,7 +85,7 @@ extern "C" {
/*========== Version =========== */
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@ -412,9 +412,12 @@ union LZ4_streamDecode_u {
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
#else
# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))

3
lib/lz4frame.c
View File

@ -897,10 +897,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
FREEMEM(dctxPtr->tmpIn);
dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
dctxPtr->maxBufferSize = bufferNeeded;
FREEMEM(dctxPtr->tmpOutBuffer);
dctxPtr->maxBufferSize = 0;
dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
dctxPtr->maxBufferSize = bufferNeeded;
} }
dctxPtr->tmpInSize = 0;
dctxPtr->tmpInTarget = 0;

10
lib/lz4frame.h
View File

@ -1,7 +1,7 @@
/*
LZ4 auto-framing library
Header File
Copyright (C) 2011-2015, Yann Collet.
Copyright (C) 2011-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@ -130,6 +130,10 @@ typedef LZ4F_frameType_t frameType_t;
typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
/* LZ4F_frameInfo_t :
* makes it possible to supply detailed frame parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
@ -139,6 +143,10 @@ typedef struct {
unsigned reserved[2]; /* must be zero for forward compatibility */
} LZ4F_frameInfo_t;
/* LZ4F_preferences_t :
* makes it possible to supply detailed compression parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */

2
lib/lz4frame_static.h
View File

@ -1,7 +1,7 @@
/*
LZ4 auto-framing library
Header File for static linking only
Copyright (C) 2011-2015, Yann Collet.
Copyright (C) 2011-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)

2
lib/lz4hc.c
View File

@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2015, Yann Collet.
Copyright (C) 2011-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)

27
programs/Makefile
View File

@ -29,18 +29,18 @@
DESTDIR ?=
PREFIX ?= /usr/local
VOID := /dev/null
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
VOID := /dev/null
CFLAGS ?= -O3 # allows custom optimization flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
# Define *.exe as extension for Windows systems
@ -52,27 +52,30 @@ endif
default: lz4
default: lz4-release
m32: lz4c32
all: lz4 lz4c
native: lz4 lz4c
all: native m32
all32: CFLAGS+=-m32
all32: all
lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
lz4-release: CFLAGS := -O3
lz4-release: lz4
lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o
$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c datagen.c
$(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
lz4c32: CFLAGS+=-m32
lz4c32: lz4
@cp lz4$(EXT) lz4c32$(EXT)
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT)
@echo Cleaning completed

2
programs/datagen.c
View File

@ -1,6 +1,6 @@
/*
datagen.c - compressible data generator test tool
Copyright (C) Yann Collet 2012-2015
Copyright (C) Yann Collet 2012-2016
GPL v2 License

2
programs/datagen.h
View File

@ -1,6 +1,6 @@
/*
datagen.h - compressible data generator header
Copyright (C) Yann Collet 2012-2015
Copyright (C) Yann Collet 2012-2016
GPL v2 License

2
programs/lz4cli.c
View File

@ -1,6 +1,6 @@
/*
LZ4cli - LZ4 Command Line Interface
Copyright (C) Yann Collet 2011-2015
Copyright (C) Yann Collet 2011-2016
GPL v2 License

35
programs/lz4io.c
View File

@ -1,6 +1,6 @@
/*
LZ4io.c - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2015
Copyright (C) Yann Collet 2011-2016
GPL v2 License
@ -34,6 +34,8 @@
* Compiler Options
**************************************/
#define _LARGE_FILES /* Large file support on 32-bits AIX */
#define _FILE_OFFSET_BITS 64 /* off_t width */
#define _LARGEFILE_SOURCE
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
@ -74,6 +76,9 @@
# define SET_SPARSE_FILE_MODE(file)
# endif
#else
# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__))
# define fseek fseeko
# endif
# define SET_BINARY_MODE(file)
# define SET_SPARSE_FILE_MODE(file)
#endif
@ -103,9 +108,6 @@
#define MIN_STREAM_BUFSIZE (192 KB)
#define LZ4IO_BLOCKSIZEID_DEFAULT 7
#define sizeT sizeof(size_t)
#define maskT (sizeT - 1)
/**************************************
* Macros
@ -623,11 +625,14 @@ static unsigned LZ4IO_readLE32 (const void* s)
return value32;
}
#define sizeT sizeof(size_t)
#define maskT (sizeT - 1)
static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
{
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
const size_t* ptrT = bufferT;
size_t bufferSizeT = bufferSize / sizeT;
size_t bufferSizeT = bufferSize / sizeT;
const size_t* const bufferTEnd = bufferT + bufferSizeT;
static const size_t segmentSizeT = (32 KB) / sizeT;
@ -716,11 +721,11 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
unsigned int blockSize;
/* Block Size */
{ size_t const sizeCheck = fread(in_buff, 1, 4, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
{ size_t const sizeCheck = fread(in_buff, 1, 4, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
/* Cannot read next block : maybe new stream ? */
g_magicRead = blockSize;
break;
@ -848,17 +853,17 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
{
size_t buffer[PTSIZET];
size_t read = 1;
size_t readBytes = 1;
unsigned long long total = MAGICNUMBER_SIZE;
unsigned storedSkips = 0;
size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
while (read) {
read = fread(buffer, 1, PTSIZE, finput);
total += read;
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, read, storedSkips);
while (readBytes) {
readBytes = fread(buffer, 1, PTSIZE, finput);
total += readBytes;
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips);
}
if (ferror(finput)) EXM_THROW(51, "Read Error")

2
programs/lz4io.h
View File

@ -1,6 +1,6 @@
/*
LZ4io.h - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2015
Copyright (C) Yann Collet 2011-2016
GPL v2 License
This program is free software; you can redistribute it and/or modify

1
tests/.gitignore vendored
View File

@ -11,3 +11,4 @@ fasttest
# test artefacts
tmp*
versionsTest

226
tests/Makefile
View File

@ -23,31 +23,28 @@
# - LZ4 source repository : https://github.com/lz4/lz4
# ##########################################################################
# fuzzer : Test tool, to check lz4 integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
# frametest : Test tool, to check lz4frame integrity on target platform
# frametest32: Same as frametest, but forced to compile in 32-bits mode
# fullbench : Precisely measure speed for each LZ4 function variant
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# datagen : generates synthetic data samples for tests & benchmarks
# ##########################################################################
DESTDIR ?=
PREFIX ?= /usr/local
VOID := /dev/null
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
LIBDIR := ../lib
PRGDIR := ../programs
VOID := /dev/null
TESTDIR := versionsTest
PYTHON ?= python3
CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
# Define *.exe as extension for Windows systems
@ -56,6 +53,7 @@ EXT =.exe
else
EXT =
endif
LZ4 := $(PRGDIR)/lz4$(EXT)
# Default test parameters
@ -64,58 +62,49 @@ FUZZER_TIME := -T3mn
NB_LOOPS ?= -i1
default: native
default: all
m32: fullbench32 fuzzer32 frametest32
native: fullbench fuzzer frametest datagen fasttest
all: native m32
all: fullbench fuzzer frametest datagen fasttest
all32: CFLAGS+=-m32
all32: all
lz4:
$(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
lz4c:
$(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
lz4c32:
$(MAKE) -C $(PRGDIR) $@
lz4c32: # create a 32-bits version for 32/64 interop tests
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
cp $(LZ4) $(LZ4)c32
fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
fullbench : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench-lib: fullbench.c $(LIBDIR)/xxhash.c
$(MAKE) -C $(LIBDIR) liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4.a
fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
fullbench-dll: fullbench.c $(LIBDIR)/xxhash.c
$(MAKE) -C $(LIBDIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LIBDIR)/dll/liblz4.dll
fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c
fuzzer : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c
frametest: $(LIBDIR)/lz4frame.o $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fasttest: $(LZ4DIR)/lz4.o fasttest.c
fasttest: $(LIBDIR)/lz4.o fasttest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
datagen : $(PRGDIR)/datagen.c datagencli.c
$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(LIBDIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
fullbench-dll$(EXT) fullbench-lib$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
@ -125,6 +114,7 @@ clean:
@rm -fR $(TESTDIR)
@echo Cleaning completed
.PHONY: versionsTest
versionsTest:
$(PYTHON) test-lz4-versions.py
@ -134,40 +124,44 @@ versionsTest:
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
MD5:=md5sum
ifneq (,$(filter $(shell uname), Darwin ))
MD5:=md5 -r
endif
test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32
test-all: test test32
test32: CFLAGS+=-m32
test32: test
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
./datagen -g5M -P100 > tmpSrc
$(PRGDIR)/lz4 -B4D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB4
$(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4
diff -s tmpSrc tmpB4
$(PRGDIR)/lz4 -B5D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB5
$(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5
diff -s tmpSrc tmpB5
$(PRGDIR)/lz4 -B6D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB6
$(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6
diff -s tmpSrc tmpB6
$(PRGDIR)/lz4 -B7D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB7
$(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7
diff -s tmpSrc tmpB7
$(PRGDIR)/lz4 tmpSrc | $(PRGDIR)/lz4 -dv --no-sparse > tmpNoSparse
$(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse
diff -s tmpSrc tmpNoSparse
ls -ls tmp*
./datagen -s1 -g1200007 -P100 | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -dv --sparse > tmpOdd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
ls -ls tmpOdd
@$(RM) tmp*
@echo "\n Compatibility with Console :"
echo "Hello World 1 !" | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -d -c
echo "Hello World 2 !" | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -d | cat
echo "Hello World 3 !" | $(PRGDIR)/lz4 --no-frame-crc | $(PRGDIR)/lz4 -d -c
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
@echo "\n Compatibility with Append :"
./datagen -P100 -g1M > tmp1M
cat tmp1M tmp1M > tmp2M
$(PRGDIR)/lz4 -B5 -v tmp1M tmpC
$(PRGDIR)/lz4 -d -v tmpC tmpR
$(PRGDIR)/lz4 -d -v tmpC >> tmpR
$(LZ4) -B5 -v tmp1M tmpC
$(LZ4) -d -v tmpC tmpR
$(LZ4) -d -v tmpC >> tmpR
ls -ls tmp*
diff tmp2M tmpR
@$(RM) tmp*
@ -175,14 +169,15 @@ test-lz4-sparse: lz4 datagen
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
./datagen -g15M > tmp
$(PRGDIR)/lz4 -v tmp | $(PRGDIR)/lz4 -t
$(PRGDIR)/lz4 -v --content-size tmp | $(PRGDIR)/lz4 -d > tmp2
$(LZ4) -v tmp | $(LZ4) -t
$(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2
diff -s tmp tmp2
# test large size [2-4] GB
@./datagen -g3G -P100 | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 --decompress --force --sparse - tmp
@./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp
@ls -ls tmp
$(PRGDIR)/lz4 --quiet --content-size tmp | $(PRGDIR)/lz4 --verbose --decompress --force --sparse - tmp2
@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2
@ls -ls tmp2
diff -s tmp tmp2
@$(RM) tmp*
test-lz4-frame-concatenation: lz4 datagen
@ -190,10 +185,10 @@ test-lz4-frame-concatenation: lz4 datagen
@echo -n > empty.test
@echo hi > nonempty.test
cat nonempty.test empty.test nonempty.test > orig.test
@$(PRGDIR)/lz4 -zq empty.test > empty.lz4.test
@$(PRGDIR)/lz4 -zq nonempty.test > nonempty.lz4.test
@$(LZ4) -zq empty.test > empty.lz4.test
@$(LZ4) -zq nonempty.test > nonempty.lz4.test
cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test
$(PRGDIR)/lz4 -d concat.lz4.test > result.test
$(LZ4) -d concat.lz4.test > result.test
sdiff orig.test result.test
@$(RM) *.test
@echo frame concatenation test completed
@ -203,54 +198,54 @@ test-lz4-multiple: lz4 datagen
@./datagen -s1 > tmp1 2> $(VOID)
@./datagen -s2 -g100K > tmp2 2> $(VOID)
@./datagen -s3 -g1M > tmp3 2> $(VOID)
$(PRGDIR)/lz4 -f -m tmp*
$(LZ4) -f -m tmp*
ls -ls tmp*
@$(RM) tmp1 tmp2 tmp3
$(PRGDIR)/lz4 -df -m *.lz4
$(LZ4) -df -m *.lz4
ls -ls tmp*
$(PRGDIR)/lz4 -f -m tmp1 notHere tmp2; echo $$?
$(LZ4) -f -m tmp1 notHere tmp2; echo $$?
@$(RM) tmp*
test-lz4-basic: lz4 datagen
@echo "\n ---- test lz4 basic compression/decompression ----"
./datagen -g0 | $(PRGDIR)/lz4 -v | $(PRGDIR)/lz4 -t
./datagen -g16KB | $(PRGDIR)/lz4 -9 | $(PRGDIR)/lz4 -t
./datagen -g0 | $(LZ4) -v | $(LZ4) -t
./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t
./datagen -g20KB > tmpSrc
$(PRGDIR)/lz4 < tmpSrc | $(PRGDIR)/lz4 -d > tmpRes
$(LZ4) < tmpSrc | $(LZ4) -d > tmpRes
diff -q tmpSrc tmpRes
$(PRGDIR)/lz4 --no-frame-crc < tmpSrc | $(PRGDIR)/lz4 -d > tmpRes
$(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes
diff -q tmpSrc tmpRes
./datagen | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -t
./datagen -g6M -P99 | $(PRGDIR)/lz4 -9BD | $(PRGDIR)/lz4 -t
./datagen -g17M | $(PRGDIR)/lz4 -9v | $(PRGDIR)/lz4 -qt
./datagen -g33M | $(PRGDIR)/lz4 --no-frame-crc | $(PRGDIR)/lz4 -t
./datagen -g256MB | $(PRGDIR)/lz4 -vqB4D | $(PRGDIR)/lz4 -t
./datagen | $(LZ4) | $(LZ4) -t
./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt
./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
@$(RM) tmp*
test-lz4-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----"
./datagen -g6GB | $(PRGDIR)/lz4 -vB5D | $(PRGDIR)/lz4 -qt
./datagen -g6GB | $(PRGDIR)/lz4 -v5BD | $(PRGDIR)/lz4 -qt
./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt
@$(RM) tmp*
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
$(PRGDIR)/lz4 -bi1
$(LZ4) -bi1
@echo "\n ---- test mode ----"
./datagen | $(PRGDIR)/lz4 -t && false || true
./datagen | $(PRGDIR)/lz4 -tf && false || true
./datagen | $(LZ4) -t && false || true
./datagen | $(LZ4) -tf && false || true
@echo "\n ---- pass-through mode ----"
./datagen | $(PRGDIR)/lz4 -d > $(VOID) && false || true
./datagen | $(PRGDIR)/lz4 -df > $(VOID)
./datagen | $(LZ4) -d > $(VOID) && false || true
./datagen | $(LZ4) -df > $(VOID)
@echo "Hello World !" > tmp1
$(PRGDIR)/lz4 -dcf tmp1
$(LZ4) -dcf tmp1
@echo "from underground..." > tmp2
$(PRGDIR)/lz4 -dcfm tmp1 tmp2
$(LZ4) -dcfm tmp1 tmp2
@echo "\n ---- test cli ----"
$(PRGDIR)/lz4 file-does-not-exist && false || true
$(PRGDIR)/lz4 -f file-does-not-exist && false || true
$(PRGDIR)/lz4 -fm file1-dne file2-dne && false || true
$(PRGDIR)/lz4 -fm file1-dne file2-dne && false || true
$(LZ4) file-does-not-exist && false || true
$(LZ4) -f file-does-not-exist && false || true
$(LZ4) -fm file1-dne file2-dne && false || true
$(LZ4) -fm file1-dne file2-dne && false || true
test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \
@ -258,53 +253,54 @@ test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
test-lz4c: lz4c datagen
@echo "\n ---- test lz4c version ----"
./datagen -g256MB | $(PRGDIR)/lz4c -l -v | $(PRGDIR)/lz4c -t
./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
test-lz4c32: CFLAGS+=-m32
test-lz4c32: test-lz4
test-interop-32-64: lz4 lz4c32 datagen
@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
./datagen -g16KB | $(PRGDIR)/lz4c32 -9 | $(PRGDIR)/lz4 -t
./datagen -P10 | $(PRGDIR)/lz4 -9B4 | $(PRGDIR)/lz4c32 -t
./datagen | $(PRGDIR)/lz4c32 | $(PRGDIR)/lz4 -t
./datagen -g1M | $(PRGDIR)/lz4 -3B5 | $(PRGDIR)/lz4c32 -t
./datagen -g256MB | $(PRGDIR)/lz4c32 -vqB4D | $(PRGDIR)/lz4 -qt
./datagen -g1G -P90 | $(PRGDIR)/lz4 | $(PRGDIR)/lz4c32 -t
./datagen -g6GB | $(PRGDIR)/lz4c32 -vq9BD | $(PRGDIR)/lz4 -qt
./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4) -t
./datagen -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t
./datagen | $(LZ4)c32 | $(LZ4) -t
./datagen -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t
./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt
./datagen -g1G -P90 | $(LZ4) | $(LZ4)c32 -t
./datagen -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt
test-lz4c32-basic: lz4c32 datagen
@echo "\n ---- test lz4c32 32-bits version ----"
./datagen -g16KB | $(PRGDIR)/lz4c32 -9 | $(PRGDIR)/lz4c32 -t
./datagen | $(PRGDIR)/lz4c32 | $(PRGDIR)/lz4c32 -t
./datagen -g256MB | $(PRGDIR)/lz4c32 -vqB4D | $(PRGDIR)/lz4c32 -qt
./datagen -g6GB | $(PRGDIR)/lz4c32 -vqB5D | $(PRGDIR)/lz4c32 -qt
./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t
./datagen | $(LZ4)c32 | $(LZ4)c32 -t
./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt
./datagen -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt
test-platform:
@echo "\n ---- test lz4 $(QEMU_SYS) platform ----"
$(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(PRGDIR)/lz4 -9 | $(QEMU_SYS) $(PRGDIR)/lz4 -t
$(QEMU_SYS) ./datagen | $(QEMU_SYS) $(PRGDIR)/lz4 | $(QEMU_SYS) $(PRGDIR)/lz4 -t
$(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(PRGDIR)/lz4 -vqB4D | $(QEMU_SYS) $(PRGDIR)/lz4 -qt
$(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t
$(QEMU_SYS) ./datagen | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t
$(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt
ifneq ($(QEMU_SYS),qemu-arm-static)
$(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(PRGDIR)/lz4 -vqB5D | $(QEMU_SYS) $(PRGDIR)/lz4 -qt
$(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt
endif
test-lz4c32: test-lz4c32-basic test-interop-32-64
test-fullbench: fullbench
./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: CFLAGS += -m32
test-fullbench32: test-fullbench
test-fuzzer: fuzzer
./fuzzer $(FUZZER_TIME)
test-fuzzer32: fuzzer32
./fuzzer32 $(FUZZER_TIME)
test-fuzzer32: CFLAGS += -m32
test-fuzzer32: test-fuzzer
test-frametest: frametest
./frametest $(FUZZER_TIME)
test-frametest32: frametest32
./frametest32 $(FUZZER_TIME)
test-frametest32: CFLAGS += -m32
test-frametest32: test-frametest
test-fasttest: fasttest
./fasttest
@ -313,17 +309,17 @@ test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -9 -BD -f tmp $(VOID)
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f tmp $(VOID)
./datagen -g16KB -s2 > tmp2
./datagen -g16KB -s3 > tmp3
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 --force --multiple tmp tmp2 tmp3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple tmp tmp2 tmp3
./datagen -g16MB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -t tmp2
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t tmp2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
./datagen -g256MB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -B4D -f -vq tmp $(VOID)
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq tmp $(VOID)
$(RM) tmp*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256

2
tests/datagencli.c
View File

@ -1,7 +1,7 @@
/*
datagencli.c
compressible data command line generator
Copyright (C) Yann Collet 2012-2015
Copyright (C) Yann Collet 2012-2016
GPL v2 License

11
tests/fasttest.c
View File

@ -51,12 +51,6 @@ int test_compress(const char *input, int inSize, char *output, int outSize)
return 0;
}
void swap(void **a, void **b) {
void *tmp = *a;
*a = *b;
*b = tmp;
}
/* Returns non-zero on failure. Not a safe function. */
int test_decompress(const char *uncompressed, const char *compressed)
{
@ -91,11 +85,10 @@ int test_decompress(const char *uncompressed, const char *compressed)
lz4StreamDecode, compressed + offset, output, unBytes);
if(bytes <= 0) return 2;
/* Check result */
{
int r = memcmp(uncompressed + unOffset, output, unBytes);
{ int const r = memcmp(uncompressed + unOffset, output, unBytes);
if (r) return 3;
}
swap((void**)&output, (void**)&lastOutput);
{ char* const tmp = output; output = lastOutput; lastOutput = tmp; }
offset += bytes;
unOffset += unBytes;
lastBytes = unBytes;

65
tests/fullbench.c
View File

@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2015
Copyright (C) Yann Collet 2012-2016
GPL v2 License
@ -229,16 +229,6 @@ static int local_LZ4_saveDict(const char* in, char* out, int inSize)
return LZ4_saveDict(&LZ4_stream, out, inSize);
}
static int local_LZ4_compress_default(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compress_default_large(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
@ -279,26 +269,6 @@ static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSi
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
}
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 1);
}
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1, 1);
}
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 1);
}
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1, 1);
}
#ifndef LZ4_DLL_IMPORT
/* declare hidden function */
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
@ -328,31 +298,16 @@ static int local_LZ4_compress_HC(const char* in, char* out, int inSize)
return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize), 9);
}
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
static int local_LZ4_compress_HC_extStateHC(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize), 9);
}
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1, 9);
}
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize)-1, 9);
}
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
static int local_LZ4_compress_HC_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1);
}
/* decompression functions */
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
@ -530,11 +485,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break;
case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break;
case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 12: compressionFunction = local_LZ4_compress_HC_extStateHC; compressorName = "LZ4_compress_HC_extStateHC"; break;
case 14: compressionFunction = local_LZ4_compress_HC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compress_HC_continue"; break;
#ifndef LZ4_DLL_IMPORT
case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
#endif
@ -549,13 +501,6 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; }
LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize);
break;
case 60: DISPLAY("Obsolete compression functions : \n"); continue;
case 61: compressionFunction = local_LZ4_compress_default; compressorName = "LZ4_compress_default"; break;
case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break;
case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break;
default :
continue; /* unknown ID : just skip */
}

15
tests/fuzzer.c
View File

@ -170,7 +170,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
static int FUZ_AddressOverflow(void)
{
char* buffers[MAX_NB_BUFF_I134+1];
int i, nbBuff=0;
int nbBuff=0;
int highAddress = 0;
DISPLAY("Overflow tests : ");
@ -185,12 +185,13 @@ static int FUZ_AddressOverflow(void)
buffers[0] = (char*)malloc(BLOCKSIZE_I134);
buffers[1] = (char*)malloc(BLOCKSIZE_I134);
if ((!buffers[0]) || (!buffers[1])) {
free(buffers[0]); free(buffers[1]);
DISPLAY("not enough memory for tests \n");
return 0;
}
for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) {
DISPLAY("%3i \b\b\b\b", nbBuff);
DISPLAY("%3i \b\b\b\b", nbBuff); fflush(stdout);
buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
if (buffers[nbBuff]==NULL) goto _endOfTests;
@ -201,7 +202,7 @@ static int FUZ_AddressOverflow(void)
}
{ size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512);
int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1);
char* const input = buffers[nbBuff-1];
char* output = buffers[nbBuff];
int r;
@ -209,15 +210,15 @@ static int FUZ_AddressOverflow(void)
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff;
{ unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
input[0] = (char)0x1F; /* Match length overflow */
input[1] = (char)0x01;
input[2] = (char)0x01;
input[3] = (char)0x00;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
output = buffers[nbBuff-2]; /* Reverse in/out pointer order */
input[0] = (char)0xF0; /* Literal length overflow */
@ -237,7 +238,7 @@ static int FUZ_AddressOverflow(void)
nbBuff++;
_endOfTests:
for (i=0 ; i<nbBuff; i++) free(buffers[i]);
{ int i; for (i=0 ; i<nbBuff; i++) free(buffers[i]); }
if (!highAddress) DISPLAY("high address not possible \n");
else DISPLAY("all overflows correctly detected \n");
return 0;

8
tests/test-lz4-versions.py
View File

@ -21,7 +21,7 @@ make_cmd = 'make'
git_cmd = 'git'
test_dat_src = 'README.md'
test_dat = 'test_dat'
head = 'r999'
head = 'v999'
def proc(cmd_args, pipe=True, dummy=False):
if dummy:
@ -43,6 +43,8 @@ def git(args, pipe=True):
def get_git_tags():
stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]'])
tags = stdout.decode('utf-8').split()
stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]'])
tags += stdout.decode('utf-8').split()
return tags
# https://stackoverflow.com/a/19711609/2132223
@ -82,11 +84,11 @@ if __name__ == '__main__':
os.chdir(clone_dir)
git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
os.chdir(r_dir + '/programs') # /path/to/lz4/lz4test/<TAG>/programs
make(['clean', 'lz4c', 'lz4c32'], False)
else:
os.chdir(programs_dir)
make(['lz4c', 'lz4c32'], False)
make(['clean', 'lz4c'], False)
shutil.copy2('lz4c', dst_lz4c)
make(['clean', 'lz4c32'], False)
shutil.copy2('lz4c32', dst_lz4c32)
# Compress test.dat by all released lz4c and lz4c32