AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #102 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet 2015-05-07 13:23:39 +02:00
commit 7d182b816a
49 changed files with 3452 additions and 1733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -10,11 +10,11 @@ before_install:
- sudo apt-get install -qq valgrind
env:
- LZ4_TRAVIS_CI_ENV=dist
- LZ4_TRAVIS_CI_ENV=travis-install
- LZ4_TRAVIS_CI_ENV=streaming-examples
- LZ4_TRAVIS_CI_ENV=cmake
- LZ4_TRAVIS_CI_ENV=clangtest
- LZ4_TRAVIS_CI_ENV=sanitize
- LZ4_TRAVIS_CI_ENV=staticAnalyze
- LZ4_TRAVIS_CI_ENV=gpptest
- LZ4_TRAVIS_CI_ENV=armtest

1
LZ4_Frame_Format.html
View File

File diff suppressed because one or more lines are too long

56
Makefile
View File

@ -31,7 +31,7 @@
# ################################################################
# Version number
export VERSION=128
export VERSION=129
export RELEASE=r$(VERSION)
DESTDIR?=
@ -41,24 +41,6 @@ LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
PRGDIR = programs
LZ4DIR = lib
DISTRIBNAME=lz4-$(RELEASE).tar.gz
TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \
$(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h \
$(LZ4DIR)/xxhash.c $(LZ4DIR)/xxhash.h \
$(LZ4DIR)/liblz4.pc.in $(LZ4DIR)/Makefile $(LZ4DIR)/LICENSE \
Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \
cmake_unofficial/CMakeLists.txt \
$(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \
$(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \
$(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \
$(PRGDIR)/bench.c $(PRGDIR)/bench.h \
$(PRGDIR)/lz4.1 \
$(PRGDIR)/Makefile $(PRGDIR)/COPYING
NONTEXT = images/image00.png images/image01.png images/image02.png \
images/image03.png images/image04.png images/image05.png \
images/image06.png
SOURCES = $(TEXT) $(NONTEXT)
# Select test target for Travis CI's Build Matrix
@ -86,7 +68,6 @@ lz4programs:
@cd $(PRGDIR); $(MAKE) -e
clean:
@rm -f $(DISTRIBNAME) *.sha1 > $(VOID)
@cd $(PRGDIR); $(MAKE) clean > $(VOID)
@cd $(LZ4DIR); $(MAKE) clean > $(VOID)
@cd examples; $(MAKE) clean > $(VOID)
@ -108,24 +89,6 @@ uninstall:
travis-install:
sudo $(MAKE) install
dist: clean
@install -dD -m 700 lz4-$(RELEASE)/lib/
@install -dD -m 700 lz4-$(RELEASE)/programs/
@install -dD -m 700 lz4-$(RELEASE)/cmake_unofficial/
@install -dD -m 700 lz4-$(RELEASE)/images/
@for f in $(TEXT); do \
tr -d '\r' < $$f > .tmp; \
install -m 600 .tmp lz4-$(RELEASE)/$$f; \
done
@rm .tmp
@for f in $(NONTEXT); do \
install -m 600 $$f lz4-$(RELEASE)/$$f; \
done
@tar -czf $(DISTRIBNAME) lz4-$(RELEASE)/
@rm -rf lz4-$(RELEASE)
@sha1sum $(DISTRIBNAME) > $(DISTRIBNAME).sha1
@echo Distribution $(DISTRIBNAME) built
test:
@cd $(PRGDIR); $(MAKE) -e test
@ -134,18 +97,21 @@ test-travis: $(TRAVIS_TARGET)
cmake:
@cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE)
gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
clangtest: clean
export CC=clang; $(MAKE) all
$(MAKE) all CC=clang CPPFLAGS="-Werror -Wconversion -Wno-sign-conversion"
sanitize: clean
$(MAKE) test CC=clang CPPFLAGS="-g -fsanitize=undefined" FUZZER_TIME="-T1mn" NB_LOOPS=-i1
staticAnalyze: clean
export CFLAGS=-g; scan-build -v $(MAKE) all
gpptest: clean
export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all
scan-build --status-bugs -v $(MAKE) all CFLAGS=-g
armtest: clean
export CC=arm-linux-gnueabi-gcc; cd lib; $(MAKE) -e all
export CC=arm-linux-gnueabi-gcc; cd programs; $(MAKE) -e bins
cd lib; $(MAKE) -e all CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror"
cd programs; $(MAKE) -e bins CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror"
streaming-examples:
cd examples; $(MAKE) -e test

42
NEWS
View File

@ -1,19 +1,31 @@
r129:
Added : LZ4_compress_fast()
Changed: New lz4 and lz4hc compression API. Previous function prototypes still supported.
Changed: Sparse file support enabled by default
New : LZ4 CLI improved performance compressing/decompressing multiple files (#86, kind contribution from Kyle J. Harper & Takayuki Matsuoka)
Fixed : GCC 4.9+ optimization bug - Reported by Markus Trippelsdorf, Greg Slazinski & Evan Nemerson
Changed: Enums converted to LZ4F_ namespace convention - by Takayuki Matsuoka
Added : AppVeyor CI environment, for Visual tests - Suggested by Takayuki Matsuoka
Modified:Obsolete functions generate warnings - Suggested by Evan Nemerson, contributed by Takayuki Matsuoka
Fixed : Bug #75 (unfinished stream), reported by Yongwoon Cho
Updated: Documentation converted to MarkDown format
r128:
New : lz4cli sparse file support
New : command -m, to compress multiple files in a single command
Fixed : Restored lz4hc compression ratio (was slightly lower since r124)
New : lz4 cli supports long commands
New : lz4frame & lz4cli frame content size support
New : lz4frame supports skippable frames
Changed:Default "make install" directory is /usr/local
New : lz4 cli supports "pass-through" mode
New : datagen can generate sparse files
New : scan-build tests
New : g++ compatibility tests
New : arm cross-compilation test
Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48)
Added : Visual project directory
Updated:Man page & Specification
New : lz4cli sparse file support (Requested by Neil Wilson, and contributed by Takayuki Matsuoka)
New : command -m, to compress multiple files in a single command (suggested by Kyle J. Harper)
Fixed : Restored lz4hc compression ratio (slightly lower since r124)
New : lz4 cli supports long commands (suggested by Takayuki Matsuoka)
New : lz4frame & lz4cli frame content size support
New : lz4frame supports skippable frames, as requested by Sergey Cherepanov
Changed: Default "make install" directory is /usr/local, as notified by Ron Johnson
New : lz4 cli supports "pass-through" mode, requested by Neil Wilson
New : datagen can generate sparse files
New : scan-build tests, thanks to kind help by Takayuki Matsuoka
New : g++ compatibility tests
New : arm cross-compilation test, thanks to kind help by Takayuki Matsuoka
Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48, reported by Thomas Klausner)
Added : Visual project directory
Updated: Man page & Specification
r127:
N/A : added a file on SVN

102
README.md
View File

@ -1,60 +1,80 @@
LZ4 - Extremely fast compression
================================
LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio.
LZ4 is lossless compression algorithm,
providing compression speed at 400 MB/s per core,
scalable with multi-cores CPU.
It also features an extremely fast decoder,
with speed in multiple GB/s per core,
typically reaching RAM speed limits on multi-core systems.
Speed can be tuned dynamically, selecting an "acceleration" factor
which trades compression ratio for more speed up.
On the other end, a high compression derivative, LZ4_HC, is also provided,
trading CPU time for improved compression ratio.
All versions feature the same excellent decompression speed.
|Branch |Status |
|------------|---------|
|master | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=master)](https://travis-ci.org/Cyan4973/lz4) |
|dev | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=dev)](https://travis-ci.org/Cyan4973/lz4) |
|master | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=master)](https://travis-ci.org/Cyan4973/lz4) [![Build status](https://ci.appveyor.com/api/projects/status/v6kxv9si529477cq/branch/master?svg=true)](https://ci.appveyor.com/project/YannCollet/lz4) |
|dev | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=dev)](https://travis-ci.org/Cyan4973/lz4) [![Build status](https://ci.appveyor.com/api/projects/status/v6kxv9si529477cq/branch/dev?svg=true)](https://ci.appveyor.com/project/YannCollet/lz4) |
> **Branch Policy:**
> - The "master" branch is considered stable, at all times.
> - The "dev" branch is the one where all contributions must be merged before being promoted to master.
> - If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted.
> - Feature branches can also exist, for dedicated testing of larger modifications before merge into "dev" branch.
> - The "dev" branch is the one where all contributions must be merged
before being promoted to master.
> + If you plan to propose a patch, please commit into the "dev" branch,
or its own feature branch.
Direct commit to "master" are not permitted.
Benchmarks
-------------------------
The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.2)](http://encode.ru/threads/1371-Filesystem-benchmark?p=33548&viewfull=1#post33548) compiled with GCC v4.6.1 on Linux Ubuntu 64-bits v11.10,
The reference system uses a Core i5-3340M @2.7GHz.
Benchmark evaluates the compression of reference [Silesia Corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia) in single-thread mode.
The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.3)]
compiled with GCC v4.8.2 on Linux Mint 64-bits v17.
The reference system uses a Core i5-4300U @1.9GHz.
Benchmark evaluates the compression of reference [Silesia Corpus]
in single-thread mode.
<table>
<tr>
<th>Compressor</th><th>Ratio</th><th>Compression</th><th>Decompression</th>
</tr>
<tr>
<th>LZ4 (r101)</th><th>2.084</th><th>422 MB/s</th><th>1820 MB/s</th>
</tr>
<tr>
<th>LZO 2.06</th><th>2.106</th><th>414 MB/s</th><th>600 MB/s</th>
</tr>
<tr>
<th>QuickLZ 1.5.1b6</th><th>2.237</th><th>373 MB/s</th><th>420 MB/s</th>
</tr>
<tr>
<th>Snappy 1.1.0</th><th>2.091</th><th>323 MB/s</th><th>1070 MB/s</th>
</tr>
<tr>
<th>LZF</th><th>2.077</th><th>270 MB/s</th><th>570 MB/s</th>
</tr>
<tr>
<th>zlib 1.2.8 -1</th><th>2.730</th><th>65 MB/s</th><th>280 MB/s</th>
</tr>
<tr>
<th>LZ4 HC (r101)</th><th>2.720</th><th>25 MB/s</th><th>2080 MB/s</th>
</tr>
<tr>
<th>zlib 1.2.8 -6</th><th>3.099</th><th>21 MB/s</th><th>300 MB/s</th>
</tr>
</table>
| Compressor | Ratio | Compression | Decompression |
| ---------- | ----- | ----------- | ------------- |
| memcpy | 1.000 | 4200 MB/s | 4200 MB/s |
|**LZ4 fast 17 (r129)**| 1.607 |**690 MB/s** | **2220 MB/s** |
|**LZ4 default (r129)**|**2.101**|**385 MB/s** | **1850 MB/s** |
| LZO 2.06 | 2.108 | 350 MB/s | 510 MB/s |
| QuickLZ 1.5.1.b6 | 2.238 | 320 MB/s | 380 MB/s |
| Snappy 1.1.0 | 2.091 | 250 MB/s | 960 MB/s |
| zlib 1.2.8 -1 | 2.730 | 59 MB/s | 250 MB/s |
|**LZ4 HC (r129)** |**2.720**| 22 MB/s | **1830 MB/s** |
| zlib 1.2.8 -6 | 3.099 | 18 MB/s | 270 MB/s |
The LZ4 block compression format is detailed within [lz4_block_format.txt](lz4_block_format.txt).
For streaming unknown amount of data and compress files of any size, a frame format has been published, and can be consulted within the file LZ4_Frame_Format.html .
Documentation
-------------------------
The raw LZ4 block compression format is detailed within [lz4_Block_format].
To compress an arbitrarily long file or data stream, multiple blocks are required.
Organizing these blocks and providing a common header format to handle their content
is the purpose of the Frame format, defined into [lz4_Frame_format].
Interoperable versions of LZ4 must respect this frame format.
Other source versions
-------------------------
Beyond the C reference source,
many contributors have created versions of lz4 in multiple languages.
A list of these sources is maintained on the [LZ4 Homepage].
[Open-Source Benchmark program by m^2 (v0.14.3)]: http://encode.ru/threads/1371-Filesystem-benchmark?p=34029&viewfull=1#post34029
[Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
[lz4_Block_format]: lz4_Block_format.md
[lz4_Frame_format]: lz4_Frame_format.md
[LZ4 Homepage]: http://www.lz4.info

13
cmake_unofficial/CMakeLists.txt
View File

@ -17,13 +17,15 @@ ENDIF()
option(BUILD_TOOLS "Build the command line tools" ON)
option(BUILD_LIBS "Build the libraries in addition to the tools" ON)
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX)
if(UNIX AND BUILD_LIBS)
add_definitions(-fPIC)
endif()
endif()
set(LZ4_DIR ../lib/)
set(PRG_DIR ../programs/)
set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c)
set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4frame.c ${LZ4_DIR}lz4frame.h ${LZ4_DIR}xxhash.c)
set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c)
if(BUILD_TOOLS AND NOT BUILD_LIBS)
@ -51,6 +53,7 @@ if(BUILD_LIBS)
install(FILES
${LZ4_DIR}/lz4.h
${LZ4_DIR}/lz4hc.h
${LZ4_DIR}/lz4frame.h
DESTINATION include
)
@ -62,7 +65,12 @@ endif()
#warnings
if(MSVC)
ADD_DEFINITIONS("-W4")
endif()
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX)
ADD_DEFINITIONS("-Wall")
endif()
if(CMAKE_COMPILER_IS_GNUCXX)
ADD_DEFINITIONS("-Wextra")
ADD_DEFINITIONS("-Wundef")
@ -70,7 +78,10 @@ ADD_DEFINITIONS("-Wshadow")
ADD_DEFINITIONS("-Wcast-align")
ADD_DEFINITIONS("-Wstrict-prototypes")
endif(CMAKE_COMPILER_IS_GNUCXX)
if((CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) AND
(NOT CMAKE_SYSTEM_NAME MATCHES "SunOS"))
ADD_DEFINITIONS("-std=c99")
endif()
ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"")
INCLUDE_DIRECTORIES (${LZ4_DIR})

25
examples/Makefile
View File

@ -1,6 +1,7 @@
# ##########################################################################
# LZ4 examples - Makefile
# Copyright (C) Yann Collet 2011-2014
#
# GPL v2 License
#
# This program is free software; you can redistribute it and/or modify
@ -21,29 +22,17 @@
# - LZ4 source repository : http://code.google.com/p/lz4/
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ##########################################################################
# lz4 : Command Line Utility, supporting gzip-like arguments
# lz4c : CLU, supporting also legacy lz4demo arguments
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# fuzzer : Test tool, to check lz4 integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
# fullbench : Precisely measure speed for each LZ4 function variant
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# This makefile compile and test
# example programs, using (mostly) LZ4 streaming library,
# kindly provided by Takayuki Matsuoka
# ##########################################################################
CC := $(CC)
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -Wno-missing-braces # Wno-missing-braces required due to GCC <4.8.3 bug
FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
TESTFILE= Makefile
LZ4DIR = ../lib
# Minimize test target for Travis CI's Build Matrix
ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
CFLAGS += -m32
else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
endif
LZ4DIR := ../lib
# Define *.exe as extension for Windows systems

17
examples/blockStreaming_doubleBuffer.c
View File

@ -2,7 +2,10 @@
// Copyright : Takayuki Matsuoka
#define _CRT_SECURE_NO_WARNINGS // for MSVC
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lz4.h"
#include <stdio.h>
@ -35,11 +38,13 @@ size_t read_bin(FILE* fp, void* array, size_t arrayBytes) {
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_stream_t lz4Stream_body = { 0 };
LZ4_stream_t lz4Stream_body;
LZ4_stream_t* lz4Stream = &lz4Stream_body;
char inpBuf[2][BLOCK_BYTES];
int inpBufIndex = 0;
LZ4_resetStream(lz4Stream);
for(;;) {
char* const inpPtr = inpBuf[inpBufIndex];
@ -50,8 +55,8 @@ void test_compress(FILE* outFp, FILE* inpFp)
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = LZ4_compress_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes);
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
if(cmpBytes <= 0) {
break;
}
@ -68,12 +73,14 @@ void test_compress(FILE* outFp, FILE* inpFp)
void test_decompress(FILE* outFp, FILE* inpFp)
{
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);
for(;;) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;

22
examples/blockStreaming_lineByLine.c
View File

@ -2,7 +2,10 @@
// Copyright : Takayuki Matsuoka
#define _CRT_SECURE_NO_WARNINGS // for MSVC
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lz4.h"
#include <stdio.h>
@ -38,8 +41,9 @@ static void test_compress(
size_t ringBufferBytes)
{
LZ4_stream_t* const lz4Stream = LZ4_createStream();
char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
char* const inpBuf = malloc(ringBufferBytes);
const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes);
char* const cmpBuf = (char*) malloc(cmpBufBytes);
char* const inpBuf = (char*) malloc(ringBufferBytes);
int inpOffset = 0;
for ( ; ; )
@ -60,8 +64,8 @@ static void test_compress(
#endif
{
const int cmpBytes = LZ4_compress_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes);
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1);
if (cmpBytes <= 0) break;
write_uint16(outFp, (uint16_t) cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
@ -86,8 +90,8 @@ static void test_decompress(
size_t ringBufferBytes)
{
LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
char* const decBuf = malloc(ringBufferBytes);
char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
char* const decBuf = (char*) malloc(ringBufferBytes);
int decOffset = 0;
for ( ; ; )
@ -121,8 +125,8 @@ static int compare(FILE* f0, FILE* f1)
{
int result = 0;
const size_t tempBufferBytes = 65536;
char* const b0 = malloc(tempBufferBytes);
char* const b1 = malloc(tempBufferBytes);
char* const b0 = (char*) malloc(tempBufferBytes);
char* const b1 = (char*) malloc(tempBufferBytes);
while(0 == result)
{

BIN
images/image00.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 2.8 KiB

BIN
images/image01.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 5.7 KiB

BIN
images/image02.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 6.1 KiB

BIN
images/image03.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 4.0 KiB

BIN
images/image04.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 3.0 KiB

BIN
images/image05.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 4.1 KiB

BIN
images/image06.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 7.8 KiB

4
lib/Makefile
View File

@ -31,7 +31,7 @@
# ################################################################
# Version numbers
VERSION ?= 128
VERSION ?= 129
LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
@ -40,7 +40,7 @@ LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
DESTDIR?=
PREFIX ?= /usr/local
CFLAGS ?= -O3
CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic
CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wcast-qual -Wstrict-prototypes -pedantic
LIBDIR?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include

641
lib/lz4.c
View File

@ -34,7 +34,7 @@
/**************************************
Tuning parameters
* Tuning parameters
**************************************/
/*
* HEAPMODE :
@ -44,50 +44,15 @@
#define HEAPMODE 0
/*
* CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS :
* By default, the source code expects the compiler to correctly optimize
* 4-bytes and 8-bytes read on architectures able to handle it efficiently.
* This is not always the case. In some circumstances (ARM notably),
* the compiler will issue cautious code even when target is able to correctly handle unaligned memory accesses.
*
* You can force the compiler to use unaligned memory access by uncommenting the line below.
* One of the below scenarios will happen :
* 1 - Your target CPU correctly handle unaligned access, and was not well optimized by compiler (good case).
* You will witness large performance improvements (+50% and up).
* Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c)
* The goal is to automatically detect such situations by adding your target CPU within an exception list.
* 2 - Your target CPU correctly handle unaligned access, and was already already optimized by compiler
* No change will be experienced.
* 3 - Your target CPU inefficiently handle unaligned access.
* You will experience a performance loss. Comment back the line.
* 4 - Your target CPU does not handle unaligned access.
* Program will crash.
* If uncommenting results in better performance (case 1)
* please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c)
* This way, an automatic detection macro can be added to match your case within later versions of the library.
* ACCELERATION_DEFAULT :
* Select the value of "acceleration" for LZ4_compress_fast() when parameter == 0
*/
/* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */
#define ACCELERATION_DEFAULT 1
/**************************************
CPU Feature Detection
* CPU Feature Detection
**************************************/
/*
* Automated efficient unaligned memory access detection
* Based on known hardware architectures
* This list will be updated thanks to feedbacks
*/
#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \
|| defined(__ARM_FEATURE_UNALIGNED) \
|| defined(__i386__) || defined(__x86_64__) \
|| defined(_M_IX86) || defined(_M_X64) \
|| defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \
|| (defined(_M_ARM) && (_M_ARM >= 7))
# define LZ4_UNALIGNED_ACCESS 1
#else
# define LZ4_UNALIGNED_ACCESS 0
#endif
/*
* LZ4_FORCE_SW_BITCOUNT
* Define this parameter if your target system or compiler does not support hardware bit count
@ -97,15 +62,15 @@
#endif
/**************************************
* Includes
**************************************/
#include "lz4.h"
/**************************************
* Compiler Options
**************************************/
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
/* "restrict" is a known keyword */
#else
# define restrict /* Disable restrict */
#endif
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h>
@ -113,7 +78,7 @@
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
#else
# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# ifdef __GNUC__
# if defined(__GNUC__) || defined(__clang__)
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
@ -123,9 +88,8 @@
# endif /* __STDC_VERSION__ */
#endif /* _MSC_VER */
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
/* LZ4_GCC_VERSION is defined into lz4.h */
#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
# define expect(expr,value) (expr)
@ -136,7 +100,7 @@
/**************************************
Memory routines
* Memory routines
**************************************/
#include <stdlib.h> /* malloc, calloc, free */
#define ALLOCATOR(n,s) calloc(n,s)
@ -146,13 +110,7 @@
/**************************************
Includes
**************************************/
#include "lz4.h"
/**************************************
Basic Types
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
@ -171,7 +129,7 @@
/**************************************
Reading and writing into memory
* Reading and writing into memory
**************************************/
#define STEPSIZE sizeof(size_t)
@ -184,10 +142,19 @@ static unsigned LZ4_isLittleEndian(void)
}
static U16 LZ4_read16(const void* memPtr)
{
U16 val16;
memcpy(&val16, memPtr, 2);
return val16;
}
static U16 LZ4_readLE16(const void* memPtr)
{
if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian()))
return *(U16*)memPtr;
if (LZ4_isLittleEndian())
{
return LZ4_read16(memPtr);
}
else
{
const BYTE* p = (const BYTE*)memPtr;
@ -197,10 +164,9 @@ static U16 LZ4_readLE16(const void* memPtr)
static void LZ4_writeLE16(void* memPtr, U16 value)
{
if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian()))
if (LZ4_isLittleEndian())
{
*(U16*)memPtr = value;
return;
memcpy(memPtr, &value, 2);
}
else
{
@ -210,41 +176,18 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
}
}
static U16 LZ4_read16(const void* memPtr)
{
if (LZ4_UNALIGNED_ACCESS)
return *(U16*)memPtr;
else
{
U16 val16;
memcpy(&val16, memPtr, 2);
return val16;
}
}
static U32 LZ4_read32(const void* memPtr)
{
if (LZ4_UNALIGNED_ACCESS)
return *(U32*)memPtr;
else
{
U32 val32;
memcpy(&val32, memPtr, 4);
return val32;
}
U32 val32;
memcpy(&val32, memPtr, 4);
return val32;
}
static U64 LZ4_read64(const void* memPtr)
{
if (LZ4_UNALIGNED_ACCESS)
return *(U64*)memPtr;
else
{
U64 val64;
memcpy(&val64, memPtr, 8);
return val64;
}
U64 val64;
memcpy(&val64, memPtr, 8);
return val64;
}
static size_t LZ4_read_ARCH(const void* p)
@ -256,31 +199,9 @@ static size_t LZ4_read_ARCH(const void* p)
}
static void LZ4_copy4(void* dstPtr, const void* srcPtr)
{
if (LZ4_UNALIGNED_ACCESS)
{
*(U32*)dstPtr = *(U32*)srcPtr;
return;
}
memcpy(dstPtr, srcPtr, 4);
}
static void LZ4_copy4(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 4); }
static void LZ4_copy8(void* dstPtr, const void* srcPtr)
{
#if GCC_VERSION!=409 /* disabled on GCC 4.9, as it generates invalid opcode (crash) */
if (LZ4_UNALIGNED_ACCESS)
{
if (LZ4_64bits())
*(U64*)dstPtr = *(U64*)srcPtr;
else
((U32*)dstPtr)[0] = ((U32*)srcPtr)[0],
((U32*)dstPtr)[1] = ((U32*)srcPtr)[1];
return;
}
#endif
memcpy(dstPtr, srcPtr, 8);
}
static void LZ4_copy8(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 8); }
/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */
static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
@ -293,7 +214,7 @@ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
/**************************************
Common Constants
* Common Constants
**************************************/
#define MINMATCH 4
@ -334,7 +255,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
@ -347,7 +268,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
@ -363,8 +284,8 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll((U64)val) >> 3);
# else
unsigned r;
if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
@ -379,8 +300,8 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz(val) >> 3);
# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
@ -422,13 +343,6 @@ static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1));
static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */
/**************************************
* Local Utils
**************************************/
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
/**************************************
* Local Structures and types
**************************************/
@ -437,7 +351,7 @@ typedef struct {
U32 currentOffset;
U32 initCheck;
const BYTE* dictionary;
const BYTE* bufferStart;
BYTE* bufferStart; /* obsolete, used for slideInputBuffer */
U32 dictSize;
} LZ4_stream_t_internal;
@ -451,6 +365,14 @@ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
/**************************************
* Local Utils
**************************************/
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
/********************************
* Compression functions
@ -464,7 +386,22 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
}
static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); }
static const U64 prime5bytes = 889523592379ULL;
static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType)
{
const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG;
const U32 hashMask = (1<<hashLog) - 1;
return ((sequence * prime5bytes) >> (40 - hashLog)) & hashMask;
}
static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType)
{
if (LZ4_64bits())
return LZ4_hashSequence64(sequence, tableType);
return LZ4_hashSequence((U32)sequence, tableType);
}
static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); }
static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
{
@ -496,15 +433,16 @@ static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t t
}
static int LZ4_compress_generic(
void* ctx,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
limitedOutput_directive outputLimited,
tableType_t const tableType,
dict_directive dict,
dictIssue_directive dictIssue)
void* const ctx,
const char* const source,
char* const dest,
const int inputSize,
const int maxOutputSize,
const limitedOutput_directive outputLimited,
const tableType_t tableType,
const dict_directive dict,
const dictIssue_directive dictIssue,
const U32 acceleration)
{
LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx;
@ -527,7 +465,7 @@ static int LZ4_compress_generic(
size_t refDelta=0;
/* Init conditions */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
switch(dict)
{
case noDict:
@ -558,15 +496,15 @@ static int LZ4_compress_generic(
BYTE* token;
{
const BYTE* forwardIp = ip;
unsigned step=1;
unsigned searchMatchNb = (1U << LZ4_skipTrigger);
unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
/* Find a match */
do {
U32 h = forwardH;
ip = forwardIp;
forwardIp += step;
step = searchMatchNb++ >> LZ4_skipTrigger;
step = (searchMatchNb++ >> LZ4_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
@ -693,13 +631,22 @@ _next_match:
_last_literals:
/* Encode Last Literals */
{
int lastRun = (int)(iend - anchor);
if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
const size_t lastRun = (size_t)(iend - anchor);
if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
if (lastRun >= RUN_MASK)
{
size_t accumulator = lastRun - RUN_MASK;
*op++ = RUN_MASK << ML_BITS;
for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
*op++ = (BYTE) accumulator;
}
else
{
*op++ = (BYTE)(lastRun<<ML_BITS);
}
memcpy(op, anchor, lastRun);
op += lastRun;
}
/* End */
@ -707,39 +654,271 @@ _last_literals:
}
int LZ4_compress(const char* source, char* dest, int inputSize)
int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
LZ4_resetStream((LZ4_stream_t*)state);
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
if (maxOutputSize >= LZ4_compressBound(inputSize))
{
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
else
{
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
}
int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
#if (HEAPMODE)
void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */
void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */
LZ4_stream_t ctx;
void* ctxPtr = &ctx;
#endif
int result;
if (inputSize < LZ4_64Klimit)
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue);
else
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue);
int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
#if (HEAPMODE)
FREEMEM(ctx);
FREEMEM(ctxPtr);
#endif
return result;
}
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize)
{
#if (HEAPMODE)
void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */
#else
U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */
#endif
int result;
return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1);
}
/* hidden debug function */
/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */
int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
LZ4_stream_t ctx;
LZ4_resetStream(&ctx);
if (inputSize < LZ4_64Klimit)
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue);
return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue);
return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
/********************************
* destSize variant
********************************/
static int LZ4_compress_destSize_generic(
void* const ctx,
const char* const src,
char* const dst,
int* const srcSizePtr,
const int targetDstSize,
const tableType_t tableType)
{
const BYTE* ip = (const BYTE*) src;
const BYTE* base = (const BYTE*) src;
const BYTE* lowLimit = (const BYTE*) src;
const BYTE* anchor = ip;
const BYTE* const iend = ip + *srcSizePtr;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
BYTE* op = (BYTE*) dst;
BYTE* const oend = op + targetDstSize;
BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */;
BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */);
BYTE* const oMaxSeq = oMaxLit - 1 /* token */;
U32 forwardH;
/* Init conditions */
if (targetDstSize < 1) return 0; /* Impossible to store anything */
if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
if (*srcSizePtr<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
*srcSizePtr = 0;
LZ4_putPosition(ip, ctx, tableType, base);
ip++; forwardH = LZ4_hashPosition(ip, tableType);
/* Main Loop */
for ( ; ; )
{
const BYTE* match;
BYTE* token;
{
const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = 1 << LZ4_skipTrigger;
/* Find a match */
do {
U32 h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> LZ4_skipTrigger);
if (unlikely(forwardIp > mflimit))
goto _last_literals;
match = LZ4_getPositionOnHash(h, ctx, tableType, base);
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
} while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
|| (LZ4_read32(match) != LZ4_read32(ip)) );
}
/* Catch up */
while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
{
/* Encode Literal length */
unsigned litLength = (unsigned)(ip - anchor);
token = op++;
if (op + ((litLength+240)/255) + litLength > oMaxLit)
{
/* Not enough space for a last match */
op--;
goto _last_literals;
}
if (litLength>=RUN_MASK)
{
unsigned len = litLength - RUN_MASK;
*token=(RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
else *token = (BYTE)(litLength<<ML_BITS);
/* Copy Literals */
LZ4_wildCopy(op, anchor, op+litLength);
op += litLength;
}
_next_match:
/* Encode Offset */
LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
/* Encode MatchLength */
{
size_t matchLength;
matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
if (op + ((matchLength+240)/255) > oMaxMatch)
{
/* Match description too long : reduce it */
matchLength = (15-1) + (oMaxMatch-op) * 255;
}
//printf("offset %5i, matchLength%5i \n", (int)(ip-match), matchLength + MINMATCH);
ip += MINMATCH + matchLength;
if (matchLength>=ML_MASK)
{
*token += ML_MASK;
matchLength -= ML_MASK;
while (matchLength >= 255) { matchLength-=255; *op++ = 255; }
*op++ = (BYTE)matchLength;
}
else *token += (BYTE)(matchLength);
}
anchor = ip;
/* Test end of block */
if (ip > mflimit) break;
if (op > oMaxSeq) break;
/* Fill table */
LZ4_putPosition(ip-2, ctx, tableType, base);
/* Test next position */
match = LZ4_getPosition(ip, ctx, tableType, base);
LZ4_putPosition(ip, ctx, tableType, base);
if ( (match+MAX_DISTANCE>=ip)
&& (LZ4_read32(match)==LZ4_read32(ip)) )
{ token=op++; *token=0; goto _next_match; }
/* Prepare next loop */
forwardH = LZ4_hashPosition(++ip, tableType);
}
_last_literals:
/* Encode Last Literals */
{
size_t lastRunSize = (size_t)(iend - anchor);
if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend)
{
/* adapt lastRunSize to fill 'dst' */
lastRunSize = (oend-op) - 1;
lastRunSize -= (lastRunSize+240)/255;
}
ip = anchor + lastRunSize;
if (lastRunSize >= RUN_MASK)
{
size_t accumulator = lastRunSize - RUN_MASK;
*op++ = RUN_MASK << ML_BITS;
for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
*op++ = (BYTE) accumulator;
}
else
{
*op++ = (BYTE)(lastRunSize<<ML_BITS);
}
memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
/* End */
*srcSizePtr = (int) (((const char*)ip)-src);
return (int) (((char*)op)-dst);
}
static int LZ4_compress_destSize_extState (void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
LZ4_resetStream((LZ4_stream_t*)state);
if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */
{
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
}
else
{
if (*srcSizePtr < LZ4_64Klimit)
return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16);
else
return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr);
}
}
int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
#if (HEAPMODE)
void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctxBody;
void* ctx = &ctxBody;
#endif
int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
#if (HEAPMODE)
FREEMEM(ctx);
@ -748,19 +927,10 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
}
/*****************************************
* Experimental : Streaming functions
*****************************************/
/*
* LZ4_initStream
* Use this function once, to init a newly allocated LZ4_stream_t structure
* Return : 1 if OK, 0 if error
*/
void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
{
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
}
/********************************
* Streaming functions
********************************/
LZ4_stream_t* LZ4_createStream(void)
{
@ -770,6 +940,11 @@ LZ4_stream_t* LZ4_createStream(void)
return lz4s;
}
void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
{
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
}
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
FREEMEM(LZ4_stream);
@ -777,6 +952,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
}
#define HASH_UNIT sizeof(size_t)
int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
{
LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
@ -784,24 +960,26 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
const BYTE* const dictEnd = p + dictSize;
const BYTE* base;
if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */
if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */
LZ4_resetStream(LZ4_dict);
if (dictSize < MINMATCH)
if (dictSize < (int)HASH_UNIT)
{
dict->dictionary = NULL;
dict->dictSize = 0;
return 0;
}
if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB;
if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
dict->currentOffset += 64 KB;
base = p - dict->currentOffset;
dict->dictionary = p;
dict->dictSize = (U32)(dictEnd - p);
dict->currentOffset += dict->dictSize;
while (p <= dictEnd-MINMATCH)
while (p <= dictEnd-HASH_UNIT)
{
LZ4_putPosition(p, dict, byU32, base);
LZ4_putPosition(p, dict->hashTable, byU32, base);
p+=3;
}
@ -830,8 +1008,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
}
FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize,
int maxOutputSize, limitedOutput_directive limit)
int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
@ -840,6 +1017,7 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so
if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */
if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
LZ4_renormDictT(streamPtr, smallest);
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
/* Check overlapping input/dictionary space */
{
@ -858,9 +1036,9 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so
{
int result;
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall);
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
else
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue);
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
streamPtr->dictSize += (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
@ -870,9 +1048,9 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so
{
int result;
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall);
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
else
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue);
result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
@ -880,18 +1058,8 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so
}
}
int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)
{
return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited);
}
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput);
}
/* Hidden debug function, to force separate dictionary mode */
/* Hidden debug function, to force external dictionary mode */
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
{
LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict;
@ -902,7 +1070,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest);
result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue);
result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
@ -955,7 +1123,7 @@ FORCE_INLINE int LZ4_decompress_generic(
)
{
/* Local Variables */
const BYTE* restrict ip = (const BYTE*) source;
const BYTE* ip = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
BYTE* op = (BYTE*) dest;
@ -1051,8 +1219,7 @@ FORCE_INLINE int LZ4_decompress_generic(
{
/* match can be copied as a single segment from external dictionary */
match = dictEnd - (lowPrefix-match);
memcpy(op, match, length);
op += length;
memmove(op, match, length); op += length;
}
else
{
@ -1110,11 +1277,11 @@ FORCE_INLINE int LZ4_decompress_generic(
if (endOnInput)
return (int) (((char*)op)-dest); /* Nb of output bytes decoded */
else
return (int) (((char*)ip)-source); /* Nb of input bytes read */
return (int) (((const char*)ip)-source); /* Nb of input bytes read */
/* Overflow error detected */
_output_error:
return (int) (-(((char*)ip)-source))-1;
return (int) (-(((const char*)ip)-source))-1;
}
@ -1138,9 +1305,9 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
typedef struct
{
BYTE* externalDict;
const BYTE* externalDict;
size_t extDictSize;
BYTE* prefixEnd;
const BYTE* prefixEnd;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
@ -1172,7 +1339,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
{
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
lz4sd->prefixSize = (size_t) dictSize;
lz4sd->prefixEnd = (BYTE*) dictionary + dictSize;
lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
lz4sd->externalDict = NULL;
lz4sd->extDictSize = 0;
return 1;
@ -1261,7 +1428,7 @@ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
}
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
@ -1277,13 +1444,21 @@ int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSi
/* debug function */
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
/***************************************************
* Obsolete Functions
***************************************************/
/* obsolete compression functions */
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
/*
These function names are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
@ -1298,23 +1473,23 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize,
int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base)
static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base)
{
MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE);
lz4ds->bufferStart = base;
}
int LZ4_resetStreamState(void* state, const char* inputBuffer)
int LZ4_resetStreamState(void* state, char* inputBuffer)
{
if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer);
LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer);
return 0;
}
void* LZ4_create (const char* inputBuffer)
void* LZ4_create (char* inputBuffer)
{
void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64);
LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer);
LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer);
return lz4ds;
}
@ -1325,32 +1500,6 @@ char* LZ4_slideInputBuffer (void* LZ4_Data)
return (char*)(ctx->bufferStart + dictSize);
}
/* Obsolete compresson functions using User-allocated state */
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
{
if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_STREAMSIZE);
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue);
else
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue);
}
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
{
if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_STREAMSIZE);
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue);
else
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue);
}
/* Obsolete streaming decompression functions */
int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)

213
lib/lz4.h
View File

@ -39,16 +39,16 @@ extern "C" {
#endif
/*
* lz4.h provides block compression functions, for optimal performance.
* lz4.h provides block compression functions, and gives full buffer control to programmer.
* If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
* please use lz4frame.h instead.
* and can let the library handle its own memory, please use lz4frame.h instead.
*/
/**************************************
* Version
**************************************/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 6 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
int LZ4_versionNumber (void);
@ -70,28 +70,32 @@ int LZ4_versionNumber (void);
* Simple Functions
**************************************/
int LZ4_compress (const char* source, char* dest, int sourceSize);
int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
LZ4_compress() :
Compresses 'sourceSize' bytes from 'source' into 'dest'.
Destination buffer must be already allocated,
and must be sized to handle worst cases situations (input data not compressible)
Worst case size evaluation is provided by function LZ4_compressBound()
inputSize : Max supported value is LZ4_MAX_INPUT_SIZE
return : the number of bytes written in buffer dest
or 0 if the compression fails
LZ4_compress_default() :
Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails
LZ4_decompress_safe() :
compressedSize : is obviously the source size
maxDecompressedSize : is the size of the destination buffer, which must be already allocated.
return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize)
If the destination buffer is not large enough, decoding will stop and output an error code (<0).
compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits,
and never writes outside of output buffer, nor reads outside of input buffer.
It is also protected against malicious data packets.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
*/
@ -99,45 +103,55 @@ LZ4_decompress_safe() :
* Advanced Functions
**************************************/
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
/*
LZ4_compressBound() :
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (output buffer size).
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
*/
int LZ4_compressBound(int isize);
int LZ4_compressBound(int inputSize);
/*
LZ4_compress_fast() :
Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "0" means "use Default value" (see lz4.c)
An acceleration value of "1" is the same as regular LZ4_compress_default()
*/
int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
/*
LZ4_compress_limitedOutput() :
Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
If it cannot achieve it, compression will stop, and result of the function will be zero.
This saves time and memory on detecting non-compressible (or barely compressible) data.
This function never writes outside of provided output buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxOutputSize : is the size of the destination buffer (which must be already allocated)
return : the number of bytes written in buffer 'dest'
or 0 if compression fails
*/
int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
/*
LZ4_compress_withState() :
Same compression functions, but using an externally allocated memory space to store compression state.
LZ4_compress_fast_extState() :
Same compression function, just using an externally allocated memory space to store compression state.
Use LZ4_sizeofState() to know how much memory must be allocated,
and then, provide it as 'void* state' to compression functions.
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
*/
int LZ4_sizeofState(void);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
/*
LZ4_compress_destSize() :
Reverse the logic, by compressing as much data as possible from 'source' buffer
into already allocated buffer 'dest' of size 'targetDestSize'.
This function either compresses the entire 'source' content into 'dest' if it's large enough,
or fill 'dest' buffer completely with as much data as possible from 'source'.
Original idea by WiredTiger team.
*sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
New value is necessarily <= old value.
return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
or 0 if compression fails
*/
int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
/*
@ -153,7 +167,6 @@ LZ4_decompress_fast() :
*/
int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
/*
LZ4_decompress_safe_partial() :
This function decompress a compressed block of size 'compressedSize' at position 'source'
@ -172,7 +185,6 @@ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedS
/***********************************************
* Streaming Compression Functions
***********************************************/
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long))
/*
@ -188,7 +200,7 @@ typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
* LZ4_resetStream
* Use this function to init an allocated LZ4_stream_t structure
*/
void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr);
void LZ4_resetStream (LZ4_stream_t* streamPtr);
/*
* LZ4_createStream will allocate and initialize an LZ4_stream_t structure
@ -197,7 +209,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr);
* They are more future proof, in case of a change of LZ4_stream_t size.
*/
LZ4_stream_t* LZ4_createStream(void);
int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr);
int LZ4_freeStream (LZ4_stream_t* streamPtr);
/*
* LZ4_loadDict
@ -206,32 +218,27 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr);
* Loading a size of 0 is allowed.
* Return : dictionary size, in bytes (necessarily <= 64 KB)
*/
int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize);
int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*
* LZ4_compress_continue
* Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
* Previous data blocks are assumed to still be present at their previous location.
* dest buffer must be already allocated, and sized to at least LZ4_compressBound(inputSize)
* LZ4_compress_fast_continue
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
* Important : Previous data blocks are assumed to still be present and unmodified !
* 'dst' buffer must be already allocated.
* If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
*/
int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
/*
* LZ4_compress_limitedOutput_continue
* Same as before, but also specify a maximum target compressed size (maxOutputSize)
* If objective cannot be met, compression exits, and returns a zero.
*/
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
/*
* LZ4_saveDict
* If previously compressed data block is not guaranteed to remain available at its memory location
* save it into a safer place (char* safeBuffer)
* Note : you don't need to call LZ4_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call again LZ4_compress_continue()
* dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
*/
int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize);
int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
/************************************************
@ -266,8 +273,18 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
If this condition is not possible, save the relevant part of decoded data into a safe buffer,
and indicate where is its new address using LZ4_setStreamDecode()
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
@ -277,8 +294,8 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue()
They are stand-alone and don't use nor update an LZ4_streamDecode_t structure.
a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
*/
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
@ -288,27 +305,55 @@ int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalS
/**************************************
* Obsolete Functions
**************************************/
/*
Obsolete decompression functions
These function names are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
- LZ4_uncompress is the same as LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
These function prototypes are now disabled; uncomment them if you really need them.
It is highly recommended to stop using these functions and migrate to newer ones */
/* Deprecate Warnings */
/* Should these warnings messages be a problem,
it is generally possible to disable them,
with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual for example.
You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# endif
#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */
/* Obsolete compression functions */
/* These functions are planned to start generate warnings by r131 approximately */
int LZ4_compress (const char* source, char* dest, int sourceSize);
int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete decompression functions */
/* These function names are completely deprecated and must no longer be used.
They are only provided here for compatibility with older programs.
- LZ4_uncompress is the same as LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
These function prototypes are now disabled; uncomment them only if you really need them.
It is highly recommended to stop using these prototypes and migrate to maintained ones */
/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
/* Obsolete streaming functions; use new streaming interface whenever possible */
void* LZ4_create (const char* inputBuffer);
int LZ4_sizeofStreamState(void);
int LZ4_resetStreamState(void* state, const char* inputBuffer);
char* LZ4_slideInputBuffer (void* state);
LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize);
int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
#if defined (__cplusplus)

339
lib/lz4frame.c
View File

@ -39,7 +39,7 @@ You can contact the author at :
/**************************************
Compiler Options
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
@ -100,16 +100,18 @@ typedef unsigned long long U64;
#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
#define LZ4F_MAGICNUMBER 0x184D2204U
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
#define LZ4F_MAXHEADERFRAME_SIZE 15
#define LZ4F_BLOCKSIZEID_DEFAULT max64KB
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
static const size_t minFHSize = 7;
static const size_t maxFHSize = 15;
static const size_t BHSize = 4;
static const int minHClevel = 3;
static const size_t minFHSize = 5;
static const U32 minHClevel = 3;
/**************************************
* Structures and local types
**************************************/
typedef struct
typedef struct LZ4F_cctx_s
{
LZ4F_preferences_t prefs;
U32 version;
@ -123,13 +125,14 @@ typedef struct
XXH32_state_t xxh;
void* lz4CtxPtr;
U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
} LZ4F_cctx_internal_t;
} LZ4F_cctx_t;
typedef struct
typedef struct LZ4F_dctx_s
{
LZ4F_frameInfo_t frameInfo;
U32 version;
U32 dStage;
U64 frameRemainingSize;
size_t maxBlockSize;
size_t maxBufferSize;
const BYTE* srcExpect;
@ -137,14 +140,14 @@ typedef struct
size_t tmpInSize;
size_t tmpInTarget;
BYTE* tmpOutBuffer;
BYTE* dict;
const BYTE* dict;
size_t dictSize;
BYTE* tmpOut;
size_t tmpOutSize;
size_t tmpOutStart;
XXH32_state_t xxh;
BYTE header[16];
} LZ4F_dctx_internal_t;
} LZ4F_dctx_t;
/**************************************
@ -156,7 +159,7 @@ static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING
unsigned LZ4F_isError(LZ4F_errorCode_t code)
{
return (code > (LZ4F_errorCode_t)(-ERROR_maxCode));
return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode));
}
const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
@ -176,7 +179,7 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID)
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
blockSizeID -= 4;
if (blockSizeID > 3) return (size_t)-ERROR_maxBlockSize_invalid;
if (blockSizeID > 3) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid;
return blockSizes[blockSizeID];
}
@ -187,7 +190,7 @@ static U32 LZ4F_readLE32 (const BYTE* srcPtr)
U32 value32 = srcPtr[0];
value32 += (srcPtr[1]<<8);
value32 += (srcPtr[2]<<16);
value32 += (srcPtr[3]<<24);
value32 += ((U32)srcPtr[3])<<24;
return value32;
}
@ -204,7 +207,7 @@ static U64 LZ4F_readLE64 (const BYTE* srcPtr)
U64 value64 = srcPtr[0];
value64 += (srcPtr[1]<<8);
value64 += (srcPtr[2]<<16);
value64 += (srcPtr[3]<<24);
value64 += ((U64)srcPtr[3]<<24);
value64 += ((U64)srcPtr[4]<<32);
value64 += ((U64)srcPtr[5]<<40);
value64 += ((U64)srcPtr[6]<<48);
@ -235,15 +238,15 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length)
/**************************************
* Simple compression functions
**************************************/
static blockSizeID_t LZ4F_optimalBSID(const blockSizeID_t requestedBSID, const size_t srcSize)
static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize)
{
blockSizeID_t proposedBSID = max64KB;
LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
size_t maxBlockSize = 64 KB;
while (requestedBSID > proposedBSID)
{
if (srcSize <= maxBlockSize)
return proposedBSID;
proposedBSID = (blockSizeID_t)((int)proposedBSID + 1);
proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1);
maxBlockSize <<= 2;
}
return requestedBSID;
@ -262,7 +265,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
prefs.autoFlush = 1;
headerSize = 15; /* header size, including magic number and frame content size*/
headerSize = maxFHSize; /* header size, including magic number and frame content size*/
streamSize = LZ4F_compressBound(srcSize, &prefs);
return headerSize + streamSize;
@ -280,7 +283,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
*/
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_cctx_internal_t cctxI;
LZ4F_cctx_t cctxI;
LZ4_stream_t lz4ctx;
LZ4F_preferences_t prefs;
LZ4F_compressOptions_t options;
@ -295,16 +298,14 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
cctxI.version = LZ4F_VERSION;
cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */
if (preferencesPtr!=NULL) prefs = *preferencesPtr;
if (preferencesPtr!=NULL)
prefs = *preferencesPtr;
else
{
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.contentSize = (U64)srcSize;
}
if (prefs.frameInfo.contentSize != 0)
prefs.frameInfo.contentSize = (U64)srcSize; /* correct content size if selected (!=0) */
prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
if (prefs.compressionLevel < minHClevel)
if (prefs.compressionLevel < (int)minHClevel)
{
cctxI.lz4CtxPtr = &lz4ctx;
cctxI.lz4CtxLevel = 1;
@ -313,12 +314,12 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
prefs.autoFlush = 1;
if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */
prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */
options.stableSrc = 1;
if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs))
return (size_t)-ERROR_dstMaxSize_tooSmall;
return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall;
errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */
if (LZ4F_isError(errorCode)) return errorCode;
@ -332,7 +333,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
if (LZ4F_isError(errorCode)) return errorCode;
dstPtr += errorCode;
if (prefs.compressionLevel >= minHClevel) /* no allocation necessary with lz4 fast */
if (prefs.compressionLevel >= (int)minHClevel) /* no allocation necessary with lz4 fast */
FREEMEM(cctxI.lz4CtxPtr);
return (dstPtr - dstStart);
@ -340,8 +341,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
/***********************************
* Advanced compression functions
* *********************************/
* Advanced compression functions
***********************************/
/* LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
@ -353,29 +354,32 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
*/
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
{
LZ4F_cctx_internal_t* cctxPtr;
LZ4F_cctx_t* cctxPtr;
cctxPtr = (LZ4F_cctx_internal_t*)ALLOCATOR(sizeof(LZ4F_cctx_internal_t));
if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-ERROR_allocation_failed);
cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t));
if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-LZ4F_ERROR_allocation_failed);
cctxPtr->version = version;
cctxPtr->cStage = 0; /* Next stage : write header */
*LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
return OK_NoError;
return LZ4F_OK_NoError;
}
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)
{
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext;
LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
FREEMEM(cctxPtr->lz4CtxPtr);
FREEMEM(cctxPtr->tmpBuff);
FREEMEM(LZ4F_compressionContext);
if (cctxPtr != NULL) /* null pointers can be safely provided to this function, like free() */
{
FREEMEM(cctxPtr->lz4CtxPtr);
FREEMEM(cctxPtr->tmpBuff);
FREEMEM(LZ4F_compressionContext);
}
return OK_NoError;
return LZ4F_OK_NoError;
}
@ -388,25 +392,25 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_preferences_t prefNull;
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
BYTE* headerStart;
size_t requiredBuffSize;
if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall;
if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC;
if (dstMaxSize < maxFHSize) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall;
if (cctxPtr->cStage != 0) return (size_t)-LZ4F_ERROR_GENERIC;
memset(&prefNull, 0, sizeof(prefNull));
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
cctxPtr->prefs = *preferencesPtr;
/* ctx Management */
{
U32 tableID = cctxPtr->prefs.compressionLevel<minHClevel ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
U32 tableID = (cctxPtr->prefs.compressionLevel < minHClevel) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
if (cctxPtr->lz4CtxLevel < tableID)
{
FREEMEM(cctxPtr->lz4CtxPtr);
if (cctxPtr->prefs.compressionLevel<minHClevel)
if (cctxPtr->prefs.compressionLevel < minHClevel)
cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
else
cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
@ -418,16 +422,16 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 128 KB);
requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
if (preferencesPtr->autoFlush)
requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 64 KB; /* just needs dict */
requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */
if (cctxPtr->maxBufferSize < requiredBuffSize)
{
cctxPtr->maxBufferSize = requiredBuffSize;
FREEMEM(cctxPtr->tmpBuff);
cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
if (cctxPtr->tmpBuff == NULL) return (size_t)-ERROR_allocation_failed;
if (cctxPtr->tmpBuff == NULL) return (size_t)-LZ4F_ERROR_allocation_failed;
}
cctxPtr->tmpIn = cctxPtr->tmpBuff;
cctxPtr->tmpInSize = 0;
@ -443,10 +447,10 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
headerStart = dstPtr;
/* FLG Byte */
*dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
+ ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */
+ (BYTE)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */
+ (BYTE)((cctxPtr->prefs.frameInfo.contentSize > 0) << 3); /* Frame content size */
+ ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */
+ ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */
/* BD Byte */
*dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
/* Optional Frame content size field */
@ -468,15 +472,16 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations.
* The LZ4F_frameInfo_t structure is optional :
* you can provide NULL as argument, all preferences will then be set to default.
* you can provide NULL as argument, preferences will then be set to cover worst case situations.
* */
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_preferences_t prefsNull;
memset(&prefsNull, 0, sizeof(prefsNull));
prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
{
const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID;
LZ4F_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID;
size_t blockSize = LZ4F_getBlockSize(bid);
unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1;
size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize;
@ -522,21 +527,21 @@ static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char*
static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level)
{
(void) level;
return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize);
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize);
}
static compressFunc_t LZ4F_selectCompression(blockMode_t blockMode, U32 level)
static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
{
if (level < minHClevel)
{
if (blockMode == blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState;
if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState;
return LZ4F_localLZ4_compress_limitedOutput_continue;
}
if (blockMode == blockIndependent) return LZ4_compressHC2_limitedOutput_withStateHC;
if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC;
return LZ4F_localLZ4_compressHC_limitedOutput_continue;
}
static int LZ4F_localSaveDict(LZ4F_cctx_internal_t* cctxPtr)
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
{
if (cctxPtr->prefs.compressionLevel < minHClevel)
return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
@ -557,7 +562,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
LZ4F_compressOptions_t cOptionsNull;
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext;
size_t blockSize = cctxPtr->maxBlockSize;
const BYTE* srcPtr = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcPtr + srcSize;
@ -567,8 +572,8 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
compressFunc_t compress;
if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC;
if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall;
if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC;
if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall;
memset(&cOptionsNull, 0, sizeof(cOptionsNull));
if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
@ -596,7 +601,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize;
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
}
}
@ -618,7 +623,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
}
/* preserve dictionary if necessary */
if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer))
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer))
{
if (compressOptionsPtr->stableSrc)
{
@ -627,13 +632,13 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
else
{
int realDictSize = LZ4F_localSaveDict(cctxPtr);
if (realDictSize==0) return (size_t)-ERROR_GENERIC;
if (realDictSize==0) return (size_t)-LZ4F_ERROR_GENERIC;
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
}
/* keep tmpIn within limits */
if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */
if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
&& !(cctxPtr->prefs.autoFlush))
{
int realDictSize = LZ4F_localSaveDict(cctxPtr);
@ -649,7 +654,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
cctxPtr->tmpInSize = sizeToCopy;
}
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled)
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
cctxPtr->totalInSize += srcSize;
@ -667,15 +672,15 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
*/
size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
compressFunc_t compress;
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC;
if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */
if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC;
if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */
(void)compressOptionsPtr; /* not yet useful */
/* select compression function */
@ -683,11 +688,11 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer,
/* compress tmp buffer */
dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
cctxPtr->tmpInSize = 0;
/* keep tmpIn within limits */
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily LZ4F_blockLinked */
{
int realDictSize = LZ4F_localSaveDict(cctxPtr);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
@ -708,7 +713,7 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer,
*/
size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
size_t errorCode;
@ -720,7 +725,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
LZ4F_writeLE32(dstPtr, 0);
dstPtr+=4; /* endMark */
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled)
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
{
U32 xxh = XXH32_digest(&(cctxPtr->xxh));
LZ4F_writeLE32(dstPtr, xxh);
@ -732,7 +737,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
if (cctxPtr->prefs.frameInfo.contentSize)
{
if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
return (size_t)-ERROR_frameSize_wrong;
return (size_t)-LZ4F_ERROR_frameSize_wrong;
}
return dstPtr - dstStart;
@ -754,23 +759,28 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
*/
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber)
{
LZ4F_dctx_internal_t* dctxPtr;
LZ4F_dctx_t* dctxPtr;
dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t));
if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC;
dctxPtr = (LZ4F_dctx_t*)ALLOCATOR(sizeof(LZ4F_dctx_t));
if (dctxPtr==NULL) return (LZ4F_errorCode_t)-LZ4F_ERROR_GENERIC;
dctxPtr->version = versionNumber;
*LZ4F_decompressionContextPtr = (LZ4F_compressionContext_t)dctxPtr;
return OK_NoError;
*LZ4F_decompressionContextPtr = (LZ4F_decompressionContext_t)dctxPtr;
return LZ4F_OK_NoError;
}
LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext)
{
LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext;
FREEMEM(dctxPtr->tmpIn);
FREEMEM(dctxPtr->tmpOutBuffer);
FREEMEM(dctxPtr);
return OK_NoError;
LZ4F_errorCode_t result = LZ4F_OK_NoError;
LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)LZ4F_decompressionContext;
if (dctxPtr != NULL) /* can accept NULL input, like free() */
{
result = (LZ4F_errorCode_t)dctxPtr->dStage;
FREEMEM(dctxPtr->tmpIn);
FREEMEM(dctxPtr->tmpOutBuffer);
FREEMEM(dctxPtr);
}
return result;
}
@ -781,8 +791,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_
typedef enum { dstage_getHeader=0, dstage_storeHeader,
dstage_getCBlockSize, dstage_storeCBlockSize,
dstage_copyDirect,
dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock,
dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut,
dstage_getCBlock, dstage_storeCBlock,
dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
dstage_decodeCBlock_intoTmp, dstage_flushOut,
dstage_getSuffix, dstage_storeSuffix,
dstage_getSFrameSize, dstage_storeSFrameSize,
dstage_skipSkippable
@ -794,8 +805,9 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader,
or an error code (testable with LZ4F_isError())
output : set internal values of dctx, such as
dctxPtr->frameInfo and dctxPtr->dStage.
input : srcVoidPtr points at the **beginning of the frame**
*/
static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize)
static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize)
{
BYTE FLG, BD, HC;
unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID;
@ -804,13 +816,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo
const BYTE* srcPtr = (const BYTE*)srcVoidPtr;
/* need to decode header to get frameInfo */
if (srcSize < minFHSize) return (size_t)-ERROR_GENERIC; /* minimal header size */
if (srcSize < minFHSize) return (size_t)-LZ4F_ERROR_frameHeader_incomplete; /* minimal frame header size */
memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
/* skippable frames */
/* special case : skippable frames */
if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
{
dctxPtr->frameInfo.frameType = skippableFrame;
dctxPtr->frameInfo.frameType = LZ4F_skippableFrame;
if (srcVoidPtr == (void*)(dctxPtr->header))
{
dctxPtr->tmpInSize = srcSize;
@ -826,7 +838,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo
}
/* control magic number */
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_frameType_unknown;
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-LZ4F_ERROR_frameType_unknown;
dctxPtr->frameInfo.frameType = LZ4F_frame;
/* Flags */
@ -838,10 +850,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo
contentChecksumFlag = (FLG>>2) & _1BIT;
/* Frame Header Size */
frameHeaderSize = contentSizeFlag ? 15 : 7;
frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize;
if (srcSize < frameHeaderSize)
{
/* not enough input to fully decode frame header */
if (srcPtr != dctxPtr->header)
memcpy(dctxPtr->header, srcPtr, srcSize);
dctxPtr->tmpInSize = srcSize;
@ -854,39 +867,39 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo
blockSizeID = (BD>>4) & _3BITS;
/* validate */
if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */
if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
if (((FLG>>0)&_2BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */
if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */
if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
if (version != 1) return (size_t)-LZ4F_ERROR_headerVersion_wrong; /* Version Number, only supported value */
if (blockChecksumFlag != 0) return (size_t)-LZ4F_ERROR_blockChecksum_unsupported; /* Not supported for the time being */
if (((FLG>>0)&_2BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */
if (((BD>>7)&_1BIT) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bit */
if (blockSizeID < 4) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; /* 4-7 only supported values for the time being */
if (((BD>>0)&_4BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */
/* check */
HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */
if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LZ4F_ERROR_headerChecksum_invalid; /* Bad header checksum error */
/* save */
dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode;
dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag;
dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID;
dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
if (contentSizeFlag)
dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
/* init */
if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
/* alloc */
bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==blockLinked) * 128 KB);
bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */
{
FREEMEM(dctxPtr->tmpIn);
FREEMEM(dctxPtr->tmpOutBuffer);
dctxPtr->maxBufferSize = bufferNeeded;
dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC;
if (dctxPtr->tmpIn == NULL) return (size_t)-LZ4F_ERROR_GENERIC;
dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize);
if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC;
if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LZ4F_ERROR_GENERIC;
}
dctxPtr->tmpInSize = 0;
dctxPtr->tmpInTarget = 0;
@ -912,28 +925,32 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo
* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress,
* or an error code which can be tested using LZ4F_isError().
*/
LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr)
LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dCtx, LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr)
{
LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext;
LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)dCtx;
if (dctxPtr->dStage == dstage_getHeader)
if (dctxPtr->dStage > dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */
{
size_t frameHeaderSize = LZ4F_decodeHeader(dctxPtr, srcBuffer, *srcSizePtr);
if (LZ4F_isError(frameHeaderSize)) return frameHeaderSize;
*srcSizePtr = frameHeaderSize; /* nb Bytes consumed */
*frameInfoPtr = dctxPtr->frameInfo; /* copy into */
dctxPtr->srcExpect = NULL;
return 4; /* nextSrcSizeHint : 4 == block header size */
size_t o=0, i=0;
/* frameInfo already decoded */
*srcSizePtr = 0;
*frameInfoPtr = dctxPtr->frameInfo;
return LZ4F_decompress(dCtx, NULL, &o, NULL, &i, NULL);
}
else
{
size_t o=0;
size_t nextSrcSize = LZ4F_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL);
if (dctxPtr->dStage <= dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */
return (size_t)-LZ4F_ERROR_frameHeader_incomplete;
*frameInfoPtr = dctxPtr->frameInfo;
return nextSrcSize;
}
/* frameInfo already decoded */
*srcSizePtr = 0;
*frameInfoPtr = dctxPtr->frameInfo;
return 0;
}
/* redirector, with common prototype */
/* trivial redirector, for common prototype */
static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
{
(void)dictStart; (void)dictSize;
@ -941,10 +958,10 @@ static int LZ4F_decompress_safe (const char* source, char* dest, int compressedS
}
static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
static void LZ4F_updateDict(LZ4F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
{
if (dctxPtr->dictSize==0)
dctxPtr->dict = (BYTE*)dstPtr; /* priority to dictionary continuity */
dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */
{
@ -954,7 +971,7 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */
{
dctxPtr->dict = (BYTE*)dstPtr0;
dctxPtr->dict = (const BYTE*)dstPtr0;
dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize;
return;
}
@ -970,7 +987,7 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
{
size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
size_t copySize = 64 KB - dctxPtr->tmpOutSize;
BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
@ -986,10 +1003,10 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */
{
size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
dctxPtr->dictSize = preserveSize;
}
memcpy(dctxPtr->dict + dctxPtr->dictSize, dstPtr, dstSize);
memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize);
dctxPtr->dictSize += dstSize;
return;
}
@ -1029,7 +1046,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* decompressOptionsPtr)
{
LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext;
LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)decompressionContext;
LZ4F_decompressOptions_t optionsNull;
const BYTE* const srcStart = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcStart + *srcSizePtr;
@ -1050,7 +1067,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
/* expect to continue decoding src buffer where it left previously */
if (dctxPtr->srcExpect != NULL)
{
if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_wrongSrcPtr;
if (srcStart != dctxPtr->srcExpect) return (size_t)-LZ4F_ERROR_srcPtr_wrong;
}
/* programmed as a state machine */
@ -1063,7 +1080,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
case dstage_getHeader:
{
if (srcEnd-srcPtr >= 7)
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) /* enough to decode - shortcut */
{
LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr);
if (LZ4F_isError(errorCode)) return errorCode;
@ -1071,7 +1088,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
break;
}
dctxPtr->tmpInSize = 0;
dctxPtr->tmpInTarget = 7;
dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */
dctxPtr->dStage = dstage_storeHeader;
}
@ -1084,7 +1101,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
srcPtr += sizeToCopy;
if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget)
{
nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4;
nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
@ -1097,10 +1114,10 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
case dstage_getCBlockSize:
{
if ((srcEnd - srcPtr) >= 4)
if ((size_t)(srcEnd - srcPtr) >= BHSize)
{
selectedIn = srcPtr;
srcPtr += 4;
srcPtr += BHSize;
}
else
{
@ -1113,15 +1130,15 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
if (dctxPtr->dStage == dstage_storeCBlockSize)
case dstage_storeCBlockSize:
{
size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
size_t sizeToCopy = BHSize - dctxPtr->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
dctxPtr->tmpInSize += sizeToCopy;
if (dctxPtr->tmpInSize < 4) /* not enough input to get full cBlockSize; wait for more */
if (dctxPtr->tmpInSize < BHSize) /* not enough input to get full cBlockSize; wait for more */
{
nextSrcSizeHint = 4 - dctxPtr->tmpInSize;
doAnotherStage=0;
nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
doAnotherStage = 0;
break;
}
selectedIn = dctxPtr->tmpIn;
@ -1135,7 +1152,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
dctxPtr->dStage = dstage_getSuffix;
break;
}
if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-ERROR_GENERIC; /* invalid cBlockSize */
if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LZ4F_ERROR_GENERIC; /* invalid cBlockSize */
dctxPtr->tmpInTarget = nextCBlockSize;
if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG)
{
@ -1145,7 +1162,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
dctxPtr->dStage = dstage_getCBlock;
if (dstPtr==dstEnd)
{
nextSrcSizeHint = nextCBlockSize + 4;
nextSrcSizeHint = nextCBlockSize + BHSize;
doAnotherStage = 0;
}
break;
@ -1158,10 +1175,10 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
memcpy(dstPtr, srcPtr, sizeToCopy);
if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameInfo.contentSize -= sizeToCopy;
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy;
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==blockLinked)
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0);
srcPtr += sizeToCopy;
@ -1172,7 +1189,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
break;
}
dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */
nextSrcSizeHint = dctxPtr->tmpInTarget + 4;
nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize;
doAnotherStage = 0;
break;
}
@ -1200,7 +1217,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
srcPtr += sizeToCopy;
if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */
{
nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4;
nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize;
doAnotherStage=0;
break;
}
@ -1223,18 +1240,18 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
int (*decoder)(const char*, char*, int, int, const char*, int);
int decodedSize;
if (dctxPtr->frameInfo.blockMode == blockLinked)
if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
decoder = LZ4_decompress_safe_usingDict;
else
decoder = LZ4F_decompress_safe;
decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
if (decodedSize < 0) return (size_t)-ERROR_GENERIC; /* decompression failed */
if (decodedSize < 0) return (size_t)-LZ4F_ERROR_GENERIC; /* decompression failed */
if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameInfo.contentSize -= decodedSize;
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==blockLinked)
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0);
dstPtr += decodedSize;
@ -1248,22 +1265,22 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
int (*decoder)(const char*, char*, int, int, const char*, int);
int decodedSize;
if (dctxPtr->frameInfo.blockMode == blockLinked)
if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
decoder = LZ4_decompress_safe_usingDict;
else
decoder = LZ4F_decompress_safe;
/* ensure enough place for tmpOut */
if (dctxPtr->frameInfo.blockMode == blockLinked)
if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
{
if (dctxPtr->dict == dctxPtr->tmpOutBuffer)
{
if (dctxPtr->dictSize > 128 KB)
{
memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB);
memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB);
dctxPtr->dictSize = 64 KB;
}
dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize;
dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize;
}
else /* dict not within tmp */
{
@ -1275,9 +1292,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
/* Decode */
decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
if (decodedSize < 0) return (size_t)-ERROR_decompressionFailed; /* decompression failed */
if (decodedSize < 0) return (size_t)-LZ4F_ERROR_decompressionFailed; /* decompression failed */
if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameInfo.contentSize -= decodedSize;
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
dctxPtr->tmpOutSize = decodedSize;
dctxPtr->tmpOutStart = 0;
dctxPtr->dStage = dstage_flushOut;
@ -1291,7 +1308,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy);
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==blockLinked)
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1);
dctxPtr->tmpOutStart += sizeToCopy;
@ -1303,7 +1320,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
dctxPtr->dStage = dstage_getCBlockSize;
break;
}
nextSrcSizeHint = 4;
nextSrcSizeHint = BHSize;
doAnotherStage = 0; /* still some data to flush */
break;
}
@ -1311,7 +1328,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
case dstage_getSuffix:
{
size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
if (dctxPtr->frameInfo.contentSize) return (size_t)-ERROR_frameSize_wrong; /* incorrect frame size decoded */
if (dctxPtr->frameRemainingSize) return (size_t)-LZ4F_ERROR_frameSize_wrong; /* incorrect frame size decoded */
if (suffixSize == 0) /* frame completed */
{
nextSrcSizeHint = 0;
@ -1352,7 +1369,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
{
U32 readCRC = LZ4F_readLE32(selectedIn);
U32 resultCRC = XXH32_digest(&(dctxPtr->xxh));
if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid;
if (readCRC != resultCRC) return (size_t)-LZ4F_ERROR_contentChecksum_invalid;
nextSrcSizeHint = 0;
dctxPtr->dStage = dstage_getHeader;
doAnotherStage = 0;
@ -1392,7 +1409,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
selectedIn = dctxPtr->header + 4;
}
/* case dstage_decodeSBlockSize: */ /* no direct access */
/* case dstage_decodeSFrameSize: */ /* no direct access */
{
size_t SFrameSize = LZ4F_readLE32(selectedIn);
dctxPtr->frameInfo.contentSize = SFrameSize;
@ -1417,7 +1434,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
}
/* preserve dictionary within tmp if necessary */
if ( (dctxPtr->frameInfo.blockMode==blockLinked)
if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
&&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
&&(!decompressOptionsPtr->stableDst)
&&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1))
@ -1427,7 +1444,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
{
size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
size_t copySize = 64 KB - dctxPtr->tmpOutSize;
BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
@ -1439,7 +1456,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
else
{
size_t newDictSize = dctxPtr->dictSize;
BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
if ((newDictSize) > 64 KB) newDictSize = 64 KB;
memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);

119
lib/lz4frame.h
View File

@ -33,7 +33,7 @@
*/
/* LZ4F is a stand-alone API to create LZ4-compressed frames
* fully conformant to specification v1.4.1.
* fully conformant to specification v1.5.1.
* All related operations, including memory management, are handled by the library.
* You don't need lz4.h when using lz4frame.h.
* */
@ -45,7 +45,7 @@ extern "C" {
#endif
/**************************************
Includes
* Includes
**************************************/
#include <stddef.h> /* size_t */
@ -62,25 +62,66 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code str
/**************************************
* Frame compression types
* ************************************/
typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7 } blockSizeID_t;
typedef enum { blockLinked=0, blockIndependent} blockMode_t;
typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t;
typedef enum { LZ4F_frame=0, skippableFrame } frameType_t;
//#define LZ4F_DISABLE_OBSOLETE_ENUMS
#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) ,x
#else
# define LZ4F_OBSOLETE_ENUM(x)
#endif
typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
LZ4F_max256KB=5,
LZ4F_max1MB=6,
LZ4F_max4MB=7
LZ4F_OBSOLETE_ENUM(max64KB = LZ4F_max64KB)
LZ4F_OBSOLETE_ENUM(max256KB = LZ4F_max256KB)
LZ4F_OBSOLETE_ENUM(max1MB = LZ4F_max1MB)
LZ4F_OBSOLETE_ENUM(max4MB = LZ4F_max4MB)
} LZ4F_blockSizeID_t;
typedef enum {
LZ4F_blockLinked=0,
LZ4F_blockIndependent
LZ4F_OBSOLETE_ENUM(blockLinked = LZ4F_blockLinked)
LZ4F_OBSOLETE_ENUM(blockIndependent = LZ4F_blockIndependent)
} LZ4F_blockMode_t;
typedef enum {
LZ4F_noContentChecksum=0,
LZ4F_contentChecksumEnabled
LZ4F_OBSOLETE_ENUM(noContentChecksum = LZ4F_noContentChecksum)
LZ4F_OBSOLETE_ENUM(contentChecksumEnabled = LZ4F_contentChecksumEnabled)
} LZ4F_contentChecksum_t;
typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame = LZ4F_skippableFrame)
} LZ4F_frameType_t;
#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
typedef LZ4F_frameType_t frameType_t;
typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
typedef struct {
blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
} LZ4F_frameInfo_t;
typedef struct {
LZ4F_frameInfo_t frameInfo;
unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
@ -91,7 +132,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
/* LZ4F_compressFrame()
* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5
* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
@ -103,9 +144,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
/**********************************
* Advanced compression functions
* ********************************/
typedef void* LZ4F_compressionContext_t;
* Advanced compression functions
**********************************/
typedef struct LZ4F_cctx_s* LZ4F_compressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */
@ -141,16 +182,18 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t cctx, void* dstBuffer, size_
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to default.
* Note that different preferences will produce in different results.
* Different preferences can produce different results.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
*/
size_t LZ4F_compressUpdate(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressUpdate()
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode)
* You can get the minimum value of dstMaxSize by using LZ4F_compressBound()
* You can get the minimum value of dstMaxSize by using LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
* LZ4F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
@ -172,18 +215,18 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t
* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
* It will flush whatever data remained within compressionContext (like LZ4_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* A successful call to LZ4F_compressEnd() makes cctx available again for future compression work.
* A successful call to LZ4F_compressEnd() makes cctx available again for next compression task.
*/
/***********************************
* Decompression functions
* *********************************/
* Decompression functions
***********************************/
typedef void* LZ4F_decompressionContext_t;
typedef struct LZ4F_dctx_s* LZ4F_decompressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
@ -202,6 +245,8 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t dctx)
* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
* The result is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
* The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
@ -211,18 +256,16 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/* LZ4F_getFrameInfo()
* This function decodes frame header information, such as blockSize.
* It is optional : you could start by calling directly LZ4F_decompress() instead.
* The objective is to extract header information without starting decompression, typically for allocation purposes.
* The function will work only if srcBuffer points at the beginning of the frame,
* and *srcSizePtr is large enough to decode the whole header (typically, between 7 & 15 bytes).
* The result is copied into an LZ4F_frameInfo_t structure, which is pointed by frameInfoPtr, and must be already allocated.
* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t.
* This function decodes frame header information (such as max blockSize, frame checksum, etc.).
* Its usage is optional : you can start by calling directly LZ4F_decompress() instead.
* The objective is to extract frame header information, typically for allocation purposes.
* LZ4F_getFrameInfo() can also be used anytime *after* starting decompression, on any valid LZ4F_decompressionContext_t.
* The result is *copied* into an existing LZ4F_frameInfo_t structure which must be already allocated.
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* It is basically the frame header size.
* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
* The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call,
* or an error code which can be tested using LZ4F_isError().
* or an error code which can be tested using LZ4F_isError()
* (typically, when there is not enough src bytes to fully decode the frame header)
* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
*/
size_t LZ4F_decompress(LZ4F_decompressionContext_t dctx,

29
lib/lz4frame_static.h
View File

@ -40,9 +40,17 @@ extern "C" {
#endif
/* lz4frame_static.h should be used solely in the context of static linking.
* It contains definitions which may still change overtime.
* Never use it in the context of DLL linking.
* */
/**************************************
* Includes
**************************************/
#include "lz4frame.h"
/**************************************
* Error management
* ************************************/
@ -50,25 +58,24 @@ extern "C" {
ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_frameType_unknown) \
ITEM(ERROR_wrongSrcPtr) \
ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_checksum_invalid) \
ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_maxCode)
#define LZ4F_GENERATE_ENUM(ENUM) ENUM,
//#define LZ4F_DISABLE_OLD_ENUMS
#ifndef LZ4F_DISABLE_OLD_ENUMS
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM,
#else
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
#endif
typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
/**************************************
Includes
**************************************/
#include "lz4frame.h"
#if defined (__cplusplus)
}
#endif

198
lib/lz4hc.c
View File

@ -1,53 +1,53 @@
/*
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2015, Yann Collet.
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
Tuning Parameter
* Tuning Parameter
**************************************/
static const int LZ4HC_compressionLevel_default = 9;
/**************************************
Includes
* Includes
**************************************/
#include "lz4hc.h"
/**************************************
Local Compiler Options
* Local Compiler Options
**************************************/
#if defined(__GNUC__)
# pragma GCC diagnostic ignored "-Wunused-function"
@ -59,18 +59,18 @@ static const int LZ4HC_compressionLevel_default = 9;
/**************************************
Common LZ4 definition
* Common LZ4 definition
**************************************/
#define LZ4_COMMONDEFS_ONLY
#include "lz4.c"
/**************************************
Local Constants
* Local Constants
**************************************/
#define DICTIONARY_LOGSIZE 16
#define MAXD (1<<DICTIONARY_LOGSIZE)
#define MAXD_MASK ((U32)(MAXD - 1))
#define MAXD_MASK (MAXD - 1)
#define HASH_LOG (DICTIONARY_LOGSIZE-1)
#define HASHTABLESIZE (1 << HASH_LOG)
@ -82,36 +82,36 @@ static const int g_maxCompressionLevel = 16;
/**************************************
Local Types
* Local Types
**************************************/
typedef struct
{
U32 hashTable[HASHTABLESIZE];
U32 hashTable[HASHTABLESIZE];
U16 chainTable[MAXD];
const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */
const BYTE* dictBase; /* alternate base for extDict */
const BYTE* inputBuffer;/* deprecated */
BYTE* inputBuffer; /* deprecated */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate;
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 compressionLevel;
} LZ4HC_Data_Structure;
/**************************************
Local Macros
* Local Macros
**************************************/
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
/**************************************
HC Compression
* HC Compression
**************************************/
static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
{
@ -119,7 +119,6 @@ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
hc4->nextToUpdate = 64 KB;
hc4->base = start - 64 KB;
hc4->inputBuffer = start;
hc4->end = start;
hc4->dictBase = start - 64 KB;
hc4->dictLimit = 64 KB;
@ -141,7 +140,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
U32 h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - HashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
chainTable[idx & 0xFFFF] = (U16)delta;
DELTANEXTU16(idx) = (U16)delta;
HashTable[h] = idx;
idx++;
}
@ -197,7 +196,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
matchIndex -= chainTable[matchIndex & 0xFFFF];
matchIndex -= DELTANEXTU16(matchIndex);
}
return (int)ml;
@ -274,7 +273,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
matchIndex -= chainTable[matchIndex & 0xFFFF];
matchIndex -= DELTANEXTU16(matchIndex);
}
return longest;
@ -536,63 +535,32 @@ _Search3:
}
int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel)
{
LZ4HC_Data_Structure ctx;
LZ4HC_init(&ctx, (const BYTE*)source);
return LZ4HC_compress_generic (&ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); }
int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
LZ4HC_Data_Structure ctx;
LZ4HC_init(&ctx, (const BYTE*)source);
return LZ4HC_compress_generic (&ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0);
}
/*****************************
* Using external allocation
* ***************************/
int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); }
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel)
int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit);
LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src);
if (maxDstSize < LZ4_compressBound(srcSize))
return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
else
return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
}
int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize)
{ return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); }
int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
LZ4HC_Data_Structure state;
return LZ4_compress_HC_extStateHC(&state, src, dst, srcSize, maxDstSize, compressionLevel);
}
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); }
/**************************************
* Streaming Functions
* ************************************/
* Streaming Functions
**************************************/
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
/* initialization */
@ -651,14 +619,15 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
}
/* Check if blocks follow each other */
if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
if ((const BYTE*)source != ctxPtr->end)
LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */
{
const BYTE* sourceEnd = (const BYTE*) source + inputSize;
const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd))
if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd))
{
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
@ -669,14 +638,12 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
}
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
if (maxOutputSize < LZ4_compressBound(inputSize))
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
else
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
}
@ -689,7 +656,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memcpy(safeBuffer, streamPtr->end - dictSize, dictSize);
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
{
U32 endIndex = (U32)(streamPtr->end - streamPtr->base);
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
@ -703,21 +670,39 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
/***********************************
* Deprecated Functions
***********************************/
* Deprecated Functions
***********************************/
/* Deprecated compression functions */
/* These functions are planned to start generate warnings by r131 approximately */
int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); }
int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); }
int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); }
int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); }
int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); }
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); }
/* Deprecated streaming functions */
/* These functions currently generate deprecation warnings */
int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
{
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer;
return 0;
}
void* LZ4_createHC (const char* inputBuffer)
void* LZ4_createHC (char* inputBuffer)
{
void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure));
LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer;
return hc4;
}
@ -727,17 +712,6 @@ int LZ4_freeHC (void* LZ4HC_Data)
return (0);
}
/*
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
}
*/
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);

211
lib/lz4hc.h
View File

@ -38,70 +38,6 @@
extern "C" {
#endif
int LZ4_compressHC (const char* source, char* dest, int inputSize);
/*
LZ4_compressHC :
return : the number of bytes in compressed buffer dest
or 0 if compression fails.
note : destination buffer must be already allocated.
To avoid any problem, size it to handle worst cases situations (input data not compressible)
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
*/
int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
/*
LZ4_compress_limitedOutput() :
Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
If it cannot achieve it, compression will stop, and result of the function will be zero.
This function never writes outside of provided output buffer.
inputSize : Max supported value is 1 GB
maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
return : the number of output bytes written in buffer 'dest'
or 0 if compression fails.
*/
int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
/*
Same functions as above, but with programmable 'compressionLevel'.
Recommended values are between 4 and 9, although any value between 0 and 16 will work.
'compressionLevel'==0 means use default 'compressionLevel' value.
Values above 16 behave the same as 16.
Equivalent variants exist for all other compression functions below.
*/
/* Note :
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
*/
/**************************************
* Using an external allocation
**************************************/
int LZ4_sizeofStateHC(void);
int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
/*
These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
To know how much memory must be allocated for the compression tables, use :
int LZ4_sizeofStateHC();
Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
The allocated memory can be provided to the compression functions using 'void* state' parameter.
LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap).
*/
/*****************************
* Includes
*****************************/
@ -109,70 +45,143 @@ They just use the externally allocated memory for state instead of allocating th
/**************************************
* Experimental Streaming Functions
* Block Compression
**************************************/
int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
LZ4_compress_HC :
Destination buffer 'dst' must be already allocated.
Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible)
Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
srcSize : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work.
0 means "use default value" (see lz4hc.c).
Values >16 behave the same as 16.
return : the number of bytes written into buffer 'dst'
or 0 if compression fails.
*/
/* Note :
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
*/
int LZ4_sizeofStateHC(void);
int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
LZ4_compress_HC_extStateHC() :
Use this function if you prefer to manually allocate memory for compression tables.
To know how much memory must be allocated for the compression tables, use :
int LZ4_sizeofStateHC();
Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
The allocated memory can then be provided to the compression functions using 'void* state' parameter.
LZ4_compress_HC_extStateHC() is equivalent to previously described function.
It just uses externally allocated memory for stateHC.
*/
/**************************************
* Streaming Compression
**************************************/
#define LZ4_STREAMHCSIZE 262192
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t;
/*
LZ4_streamHC_t
This structure allows static allocation of LZ4 HC streaming state.
State must then be initialized using LZ4_resetStreamHC() before first use.
LZ4_streamHC_t
This structure allows static allocation of LZ4 HC streaming state.
State must then be initialized using LZ4_resetStreamHC() before first use.
Static allocation should only be used with statically linked library.
If you want to use LZ4 as a DLL, please use construction functions below, which are more future-proof.
Static allocation should only be used in combination with static linking.
If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof.
*/
LZ4_streamHC_t* LZ4_createStreamHC(void);
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr);
int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
/*
These functions create and release memory for LZ4 HC streaming state.
Newly created states are already initialized.
Existing state space can be re-used anytime using LZ4_resetStreamHC().
If you use LZ4 as a DLL, please use these functions instead of direct struct allocation,
to avoid size mismatch between different versions.
These functions create and release memory for LZ4 HC streaming state.
Newly created states are already initialized.
Existing state space can be re-used anytime using LZ4_resetStreamHC().
If you use LZ4 as a DLL, use these functions instead of static structure allocation,
to avoid size mismatch between different versions.
*/
void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize);
void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize);
int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
/*
These functions compress data in successive blocks of any size, using previous blocks as dictionary.
One key assumption is that each previous block will remain read-accessible while compressing next block.
These functions compress data in successive blocks of any size, using previous blocks as dictionary.
One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
There is an exception for ring buffers, which can be smaller 64 KB.
Such case is automatically detected and correctly handled by LZ4_compress_HC_continue().
Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block.
They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
Then, use LZ4_compress_HC_continue() to compress each successive block.
It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation.
If, for any reason, previous data block can't be preserved in memory during next compression block,
you must save it to a safer memory space,
using LZ4_saveDictHC().
If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
you must save it to a safer memory space, using LZ4_saveDictHC().
Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
*/
/**************************************
* Deprecated Streaming Functions
* ************************************/
/* Note : these streaming functions follows the older model, and should no longer be used */
void* LZ4_createHC (const char* inputBuffer);
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
int LZ4_freeHC (void* LZ4HC_Data);
* Deprecated Functions
**************************************/
/* Deprecate Warnings */
/* Should these warnings messages be a problem,
it is generally possible to disable them,
with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual for example.
You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# endif
#endif // LZ4_DEPRECATE_WARNING_DEFBLOCK
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
/* compression functions */
/* these functions are planned to trigger warning messages by r131 approximately */
int LZ4_compressHC (const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_sizeofStreamStateHC(void);
int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
/* Streaming functions following the older model; should no longer be used */
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
#if defined (__cplusplus)

183
lib/xxhash.c
View File

@ -29,13 +29,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
- public discussion board : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
* Tuning parameters
***************************************/
**************************************/
/* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
@ -93,10 +92,7 @@ static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
/* for memcpy() */
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size)
{
return memcpy(dest,src,size);
}
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
/**************************************
@ -104,51 +100,36 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size)
***************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS)
# define _PACKED __attribute__ ((packed))
#else
# define _PACKED
#endif
#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
# ifdef __IBMC__
# pragma pack(1)
# else
# pragma pack(push, 1)
# endif
#endif
typedef struct _U32_S
static U32 XXH_read32(const void* memPtr)
{
U32 v;
} _PACKED U32_S;
typedef struct _U64_S
U32 val32;
memcpy(&val32, memPtr, 4);
return val32;
}
static U64 XXH_read64(const void* memPtr)
{
U64 v;
} _PACKED U64_S;
#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
# pragma pack(pop)
#endif
#define A32(x) (((U32_S *)(x))->v)
#define A64(x) (((U64_S *)(x))->v)
U64 val64;
memcpy(&val64, memPtr, 8);
return val64;
}
/*****************************************
/******************************************
* Compiler-specific Functions and Macros
******************************************/
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
@ -190,7 +171,55 @@ static U64 XXH_swap64 (U64 x)
#endif
/**************************************
/***************************************
* Architecture Macros
***************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */
static const int one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one))
#endif
/*****************************
* Memory reads
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
else
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
else
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
}
FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
/***************************************
* Macros
***************************************/
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
/***************************************
* Constants
***************************************/
#define PRIME32_1 2654435761U
@ -206,55 +235,7 @@ static U64 XXH_swap64 (U64 x)
#define PRIME64_5 2870177450012600261ULL
/***************************************
* Architecture Macros
****************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */
static const int one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one))
#endif
/**************************************
* Macros
***************************************/
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
/****************************
* Memory reads
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr));
else
return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr));
else
return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr);
}
FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
/****************************
/*****************************
* Simple Hash Functions
*****************************/
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
@ -334,7 +315,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH
}
unsigned int XXH32 (const void* input, size_t len, unsigned seed)
unsigned XXH32 (const void* input, size_t len, unsigned seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
@ -346,7 +327,7 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed)
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 3) == 0) /* Input is aligned, let's leverage the speed advantage */
if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
@ -503,7 +484,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed
}
/****************************************************
* Advanced Hash Functions
* Advanced Hash Functions
****************************************************/
/*** Allocation ***/
@ -687,9 +668,9 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
{
XXH_istate32_t* state = (XXH_istate32_t*) state_in;
const XXH_istate32_t* state = (const XXH_istate32_t*) state_in;
const BYTE * p = (const BYTE*)state->mem32;
BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize;
const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
if (state->total_len >= 16)
@ -841,9 +822,9 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
{
XXH_istate64_t * state = (XXH_istate64_t *) state_in;
const XXH_istate64_t * state = (const XXH_istate64_t *) state_in;
const BYTE * p = (const BYTE*)state->mem64;
BYTE* bEnd = (BYTE*)state->mem64 + state->memsize;
const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
if (state->total_len >= 32)

6
lib/xxhash.h
View File

@ -29,7 +29,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : http://code.google.com/p/xxhash/
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* Notice extracted from xxHash homepage :
@ -57,8 +57,8 @@ Q.Score is a measure of quality of the hash function.
It depends on successfully passing SMHasher test set.
10 is a perfect score.
A new 64-bits version, named XXH64, is available since r35.
It offers better speed for 64-bits applications.
A 64-bits version, named XXH64, is available since r35.
It offers much better speed, but for 64-bits applications only.
Name Speed on 64 bits Speed on 32 bits
XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s

43
lz4_block_format.txt → lz4_Block_format.md
View File

@ -1,10 +1,9 @@
LZ4 Block Format Description
============================
Last revised: 2015-03-26;
Last revised: 2015-03-26.
Author : Yann Collet
This small specification intents to provide enough information
to anyone willing to produce LZ4-compatible compressed data blocks
using any programming language.
@ -26,7 +25,8 @@ on implementation details of the compressor, and vice versa.
Compressed block format
-----------------------
An LZ4 compressed block is composed of sequences.
Schematically, a sequence is a suite of literals, followed by a match copy.
A sequence is a suite of literals (not-compressed bytes),
followed by a match copy.
Each sequence starts with a token.
The token is a one byte value, separated into two 4-bits fields.
@ -35,27 +35,30 @@ Therefore each field ranges from 0 to 15.
The first field uses the 4 high-bits of the token.
It provides the length of literals to follow.
(Note : a literal is a not-compressed byte).
If the field value is 0, then there is no literal.
If it is 15, then we need to add some more bytes to indicate the full length.
Each additionnal byte then represent a value from 0 to 255,
Each additional byte then represent a value from 0 to 255,
which is added to the previous value to produce a total length.
When the byte value is 255, another byte is output.
There can be any number of bytes following the token. There is no "size limit".
(Sidenote this is why a not-compressible input block is expanded by 0.4%).
(Side note : this is why a not-compressible input block is expanded by 0.4%).
Example 1 : A length of 48 will be represented as :
- 15 : value for the 4-bits High field
- 33 : (=48-15) remaining length to reach 48
- 15 : value for the 4-bits High field
- 33 : (=48-15) remaining length to reach 48
Example 2 : A length of 280 will be represented as :
- 15 : value for the 4-bits High field
- 255 : following byte is maxed, since 280-15 >= 255
- 10 : (=280 - 15 - 255) ) remaining length to reach 280
- 15 : value for the 4-bits High field
- 255 : following byte is maxed, since 280-15 >= 255
- 10 : (=280 - 15 - 255) ) remaining length to reach 280
Example 3 : A length of 15 will be represented as :
- 15 : value for the 4-bits High field
- 0 : (=15-15) yes, the zero must be output
- 15 : value for the 4-bits High field
- 0 : (=15-15) yes, the zero must be output
Following the token and optional length bytes, are the literals themselves.
They are exactly as numerous as previously decoded (length of literals).
@ -65,7 +68,8 @@ It's possible that there are zero literal.
Following the literals is the match copy operation.
It starts by the offset.
This is a 2 bytes value, in little endian format.
This is a 2 bytes value, in little endian format
(the 1st byte is the "low" byte, the 2nd one is the "high" byte).
The offset represents the position of the match to be copied from.
1 means "current position - 1 byte".
@ -83,7 +87,7 @@ we output additional bytes, one at a time, with values ranging from 0 to 255.
They are added to total to provide the final match length.
A 255 value means there is another byte to read and add.
There is no limit to the number of optional bytes that can be output this way.
(This points towards a maximum achievable compression ratio of ~250).
(This points towards a maximum achievable compression ratio of about 250).
With the offset and the matchlength,
the decoder can now proceed to copy the data from the already decoded buffer.
@ -95,9 +99,11 @@ Parsing restrictions
-----------------------
There are specific parsing rules to respect in order to remain compatible
with assumptions made by the decoder :
1) The last 5 bytes are always literals
2) The last match must start at least 12 bytes before end of block
Consequently, a block with less than 13 bytes cannot be compressed.
1. The last 5 bytes are always literals
2. The last match must start at least 12 bytes before end of block.
Consequently, a block with less than 13 bytes cannot be compressed.
These rules are in place to ensure that the decoder
will never read beyond the input buffer, nor write beyond the output buffer.
@ -118,4 +124,3 @@ or full optimal parsing.
All these trade-off offer distinctive speed/memory/compression advantages.
Whatever the method used by the compressor, its result will be decodable
by any LZ4 decoder if it follows the format specification described above.

385
lz4_Frame_format.md Normal file
View File

@ -0,0 +1,385 @@
LZ4 Frame Format Description
============================
###Notices
Copyright (c) 2013-2015 Yann Collet
Permission is granted to copy and distribute this document
for any purpose and without charge,
including translations into other languages
and incorporation into compilations,
provided that the copyright notice and this notice are preserved,
and that any substantive changes or deletions from the original
are clearly marked.
Distribution of this document is unlimited.
###Version
1.5.1 (31/03/2015)
Introduction
------------
The purpose of this document is to define a lossless compressed data format,
that is independent of CPU type, operating system,
file system and character set, suitable for
File compression, Pipe and streaming compression
using the [LZ4 algorithm](http://www.lz4.info).
The data can be produced or consumed,
even for an arbitrarily long sequentially presented input data stream,
using only an a priori bounded amount of intermediate storage,
and hence can be used in data communications.
The format uses the LZ4 compression method,
and optional [xxHash-32 checksum method](https://github.com/Cyan4973/xxHash),
for detection of data corruption.
The data format defined by this specification
does not attempt to allow random access to compressed data.
This specification is intended for use by implementers of software
to compress data into LZ4 format and/or decompress data from LZ4 format.
The text of the specification assumes a basic background in programming
at the level of bits and other primitive data representations.
Unless otherwise indicated below,
a compliant compressor must produce data sets
that conform to the specifications presented here.
It doesn’t need to support all options though.
A compliant decompressor must be able to decompress
at least one working set of parameters
that conforms to the specifications presented here.
It may also ignore checksums.
Whenever it does not support a specific parameter within the compressed stream,
it must produce a non-ambiguous error code
and associated error message explaining which parameter is unsupported.
General Structure of LZ4 Frame format
-------------------------------------
| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum |
|:-------:|:-------------:| ----- | ----- | ------- | ----------- |
| 4 bytes | 3-11 bytes | | | 4 bytes | 4 bytes |
__Magic Number__
4 Bytes, Little endian format.
Value : 0x184D2204
__Frame Descriptor__
3 to 11 Bytes, to be detailed in the next part.
Most important part of the spec.
__Data Blocks__
To be detailed later on.
That’s where compressed data is stored.
__EndMark__
The flow of blocks ends when the last data block has a size of “0”.
The size is expressed as a 32-bits value.
__Content Checksum__
Content Checksum verify that the full content has been decoded correctly.
The content checksum is the result
of [xxh32() hash function](https://github.com/Cyan4973/xxHash)
digesting the original (decoded) data as input, and a seed of zero.
Content checksum is only present when its associated flag
is set in the frame descriptor.
Content Checksum validates the result,
that all blocks were fully transmitted in the correct order and without error,
and also that the encoding/decoding process itself generated no distortion.
Its usage is recommended.
__Frame Concatenation__
In some circumstances, it may be preferable to append multiple frames,
for example in order to add new data to an existing compressed file
without re-framing it.
In such case, each frame has its own set of descriptor flags.
Each frame is considered independent.
The only relation between frames is their sequential order.
The ability to decode multiple concatenated frames
within a single stream or file
is left outside of this specification.
As an example, the reference lz4 command line utility behavior is
to decode all concatenated frames in their sequential order.
Frame Descriptor
----------------
| FLG | BD | (Content Size) | HC |
| ------- | ------- |:--------------:| ------- |
| 1 byte | 1 byte | 0 - 8 bytes | 1 byte |
The descriptor uses a minimum of 3 bytes,
and up to 11 bytes depending on optional parameters.
__FLG byte__
| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 |
| ------- | ------- | ------- | --------- | ------- | --------- | -------- |
|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*|
__BD byte__
| BitNb | 7 | 6-5-4 | 3-2-1-0 |
| ------- | -------- | ------------ | -------- |
|FieldName|*Reserved*| Block MaxSize|*Reserved*|
In the tables, bit 7 is highest bit, while bit 0 is lowest.
__Version Number__
2-bits field, must be set to “01”.
Any other value cannot be decoded by this version of the specification.
Other version numbers will use different flag layouts.
__Block Independence flag__
If this flag is set to “1”, blocks are independent.
If this flag is set to “0”, each block depends on previous ones
(up to LZ4 window size, which is 64 KB).
In such case, it’s necessary to decode all blocks in sequence.
Block dependency improves compression ratio, especially for small blocks.
On the other hand, it makes direct jumps or multi-threaded decoding impossible.
__Block checksum flag__
If this flag is set, each data block will be followed by a 4-bytes checksum,
calculated by using the xxHash-32 algorithm on the raw (compressed) data block.
The intention is to detect data corruption (storage or transmission errors)
immediately, before decoding.
Block checksum usage is optional.
__Content Size flag__
If this flag is set, the uncompressed size of data included within the frame
will be present as an 8 bytes unsigned little endian value, after the flags.
Content Size usage is optional.
__Content checksum flag__
If this flag is set, a content checksum will be appended after the EndMark.
Recommended value : “1” (content checksum is present)
__Block Maximum Size__
This information is intended to help the decoder allocate memory.
Size here refers to the original (uncompressed) data size.
Block Maximum Size is one value among the following table :
| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
| --- | --- | --- | --- | ----- | ------ | ---- | ---- |
| N/A | N/A | N/A | N/A | 64 KB | 256 KB | 1 MB | 4 MB |
The decoder may refuse to allocate block sizes above a (system-specific) size.
Unused values may be used in a future revision of the spec.
A decoder conformant to the current version of the spec
is only able to decode blocksizes defined in this spec.
__Reserved bits__
Value of reserved bits **must** be 0 (zero).
Reserved bit might be used in a future version of the specification,
typically enabling new optional features.
If this happens, a decoder respecting the current version of the specification
shall not be able to decode such a frame.
__Content Size__
This is the original (uncompressed) size.
This information is optional, and only present if the associated flag is set.
Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.
Format is Little endian.
This value is informational.
It can be safely skipped by a conformant decoder.
__Header Checksum__
One-byte checksum of combined descriptor fields, including optional ones.
The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF } `
using zero as a seed,
and the full Frame Descriptor as an input
(including optional fields when they are present).
A wrong checksum indicates an error in the descriptor.
Header checksum is informational and can be skipped.
Data Blocks
-----------
| Block Size | data | (Block Checksum) |
|:----------:| ------ |:----------------:|
| 4 bytes | | 0 - 4 bytes |
__Block Size__
This field uses 4-bytes, format is little-endian.
The highest bit is “1” if data in the block is uncompressed.
The highest bit is “0” if data in the block is compressed by LZ4.
All other bits give the size, in bytes, of the following data block
(the size does not include the block checksum if present).
Block Size shall never be larger than Block Maximum Size.
Such a thing could happen for incompressible source data.
In such case, such a data block shall be passed in uncompressed format.
__Data__
Where the actual data to decode stands.
It might be compressed or not, depending on previous field indications.
Uncompressed size of Data can be any size, up to “block maximum size”.
Note that data block is not necessarily full :
an arbitrary “flush” may happen anytime. Any block can be “partially filled”.
__Block checksum__
Only present if the associated flag is set.
This is a 4-bytes checksum value, in little endian format,
calculated by using the xxHash-32 algorithm on the raw (undecoded) data block,
and a seed of zero.
The intention is to detect data corruption (storage or transmission errors)
before decoding.
Block checksum is cumulative with Content checksum.
Skippable Frames
----------------
| Magic Number | Frame Size | User Data |
|:------------:|:----------:| --------- |
| 4 bytes | 4 bytes | |
Skippable frames allow the integration of user-defined data
into a flow of concatenated frames.
Its design is pretty straightforward,
with the sole objective to allow the decoder to quickly skip
over user-defined data and continue decoding.
For the purpose of facilitating identification,
it is discouraged to start a flow of concatenated frames with a skippable frame.
If there is a need to start such a flow with some user data
encapsulated into a skippable frame,
it’s recommended to start with a zero-byte LZ4 frame
followed by a skippable frame.
This will make it easier for file type identifiers.
__Magic Number__
4 Bytes, Little endian format.
Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F.
All 16 values are valid to identify a skippable frame.
__Frame Size__
This is the size, in bytes, of the following User Data
(without including the magic number nor the size field itself).
4 Bytes, Little endian format, unsigned 32-bits.
This means User Data can’t be bigger than (2^32-1) Bytes.
__User Data__
User Data can be anything. Data will just be skipped by the decoder.
Legacy frame
------------
The Legacy frame format was defined into the initial versions of “LZ4Demo”.
Newer compressors should not use this format anymore, as it is too restrictive.
Main characteristics of the legacy format :
- Fixed block size : 8 MB.
- All blocks must be completely filled, except the last one.
- All blocks are always compressed, even when compression is detrimental.
- The last block is detected either because
it is followed by the “EOF” (End of File) mark,
or because it is followed by a known Frame Magic Number.
- No checksum
- Convention is Little endian
| MagicNb | B.CSize | CData | B.CSize | CData | (...) | EndMark |
| ------- | ------- | ----- | ------- | ----- | ------- | ------- |
| 4 bytes | 4 bytes | CSize | 4 bytes | CSize | x times | EOF |
__Magic Number__
4 Bytes, Little endian format.
Value : 0x184C2102
__Block Compressed Size__
This is the size, in bytes, of the following compressed data block.
4 Bytes, Little endian format.
__Data__
Where the actual compressed data stands.
Data is always compressed, even when compression is detrimental.
__EndMark__
End of legacy frame is implicit only.
It must be followed by a standard EOF (End Of File) signal,
wether it is a file or a stream.
Alternatively, if the frame is followed by a valid Frame Magic Number,
it is considered completed.
It makes legacy frames compatible with frame concatenation.
Any other value will be interpreted as a block size,
and trigger an error if it does not fit within acceptable range.
Version changes
---------------
1.5.1 : changed format to MarkDown compatible
1.5 : removed Dictionary ID from specification
1.4.1 : changed wording from “stream” to “frame”
1.4 : added skippable streams, re-added stream checksum
1.3 : modified header checksum
1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”.
1.1 : optional fields are now part of the descriptor
1.0 : changed “block size” specification, adding a compressed/uncompressed flag
0.9 : reduced scale of “block maximum size” table
0.8 : removed : high compression flag
0.7 : removed : stream checksum
0.6 : settled : stream size uses 8 bytes, endian convention is little endian
0.5: added copyright notice
0.4 : changed format to Google Doc compatible OpenDocument

118
programs/Makefile
View File

@ -27,24 +27,24 @@
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# fuzzer : Test tool, to check lz4 integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
# frametest : Test tool, to check lz4frame integrity on target platform
# frametest32: Same as frametest, but forced to compile in 32-bits mode
# fullbench : Precisely measure speed for each LZ4 function variant
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# datagen : generates synthetic data samples for tests & benchmarks
# ##########################################################################
RELEASE?= r128
RELEASE?= r129
DESTDIR?=
PREFIX ?= /usr/local
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
BINDIR=$(PREFIX)/bin
MANDIR=$(PREFIX)/share/man/man1
LZ4DIR=../lib
TEST_FILES = COPYING
TEST_TARGETS=test-native
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
# Define *.exe as extension for Windows systems
@ -58,7 +58,10 @@ endif
# Select test target for Travis CI's Build Matrix
TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV)
TRAVIS_TARGET:= $(LZ4_TRAVIS_CI_ENV)
TEST_FILES := COPYING
TEST_TARGETS := test-native
FUZZER_TIME := -T9mn
default: lz4
@ -100,7 +103,7 @@ datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
@rm -f core *.o *.test \
@rm -f core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) \
@ -148,15 +151,18 @@ test-travis: $(TRAVIS_TARGET)
test-lz4-sparse: lz4 datagen
@echo ---- test sparse file support ----
./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dv --sparse > tmpB4
./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dv --sparse > tmpB5
./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dv --sparse > tmpB6
./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dv --sparse > tmpB7
./datagen -g5M -P100 > tmpSrc
./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4
diff -s tmpSrc tmpB4
./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5
diff -s tmpSrc tmpB5
./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6
diff -s tmpSrc tmpB6
./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7
diff -s tmpSrc tmpB7
./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse
diff -s tmpSrc tmpNoSparse
ls -ls tmp*
./datagen -g50M -P100 | diff -s - tmpB4
./datagen -g50M -P100 | diff -s - tmpB5
./datagen -g50M -P100 | diff -s - tmpB6
./datagen -g50M -P100 | diff -s - tmpB7
./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
ls -ls tmpOdd
@ -168,6 +174,11 @@ test-lz4-contentSize: lz4 datagen
./lz4 -v tmp | ./lz4 -t
./lz4 -v --content-size tmp | ./lz4 -d > tmp2
diff -s tmp tmp2
# test large size [2-4] GB
@./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp
@ls -ls tmp
./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2
@ls -ls tmp2
@rm tmp*
test-lz4-frame-concatenation: lz4 datagen
@ -183,24 +194,30 @@ test-lz4-frame-concatenation: lz4 datagen
@rm *.test
@echo frame concatenation test completed
test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
test-lz4-multiple: lz4 datagen
@echo ---- test multiple files ----
@./datagen -s1 > tmp1 2> $(VOID)
@./datagen -s2 -g100K > tmp2 2> $(VOID)
@./datagen -s3 -g1M > tmp3 2> $(VOID)
./lz4 -f -m tmp*
ls -ls tmp*
rm tmp1 tmp2 tmp3
./lz4 -df -m *.lz4
ls -ls tmp*
./lz4 -f -m tmp1 notHere tmp2; echo $$?
@rm tmp*
test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
@echo ---- test lz4 basic compression/decompression ----
./datagen -g0 | ./lz4 -v | ./lz4 -t
./datagen -g16KB | ./lz4 -9 | ./lz4 -t
./datagen | ./lz4 | ./lz4 -t
./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t
./datagen -g17M | ./lz4 -9v | ./lz4 -tq
./datagen -g17M | ./lz4 -9v | ./lz4 -qt
./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -t
./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -t
@echo ---- test multiple input files ----
@./datagen -s1 > file1
@./datagen -s2 > file2
@./datagen -s3 > file3
./lz4 -f -m file1 file2 file3
ls -l file*
@rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt
./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt
@echo ---- test pass-through ----
./datagen | ./lz4 -tf
@ -212,41 +229,46 @@ test-lz4c32: lz4 lz4c32 datagen
./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t
./datagen | ./lz4c32 | ./lz4c32 -t
./datagen | ./lz4c32 | ./lz4 -t
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -t
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -t
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -t
./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -t
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt
./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt
test-fullbench: fullbench
./fullbench --no-prompt $(TEST_FILES)
./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(TEST_FILES)
./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fuzzer: fuzzer
./fuzzer
./fuzzer $(FUZZER_TIME)
test-fuzzer32: fuzzer32
./fuzzer32
./fuzzer32 $(FUZZER_TIME)
test-frametest: frametest
./frametest
./frametest $(FUZZER_TIME)
test-frametest32: frametest32
./frametest32
./frametest32 $(FUZZER_TIME)
test-mem: lz4 datagen fuzzer frametest
valgrind --leak-check=yes ./datagen -g50M > $(VOID)
test-mem: lz4 datagen fuzzer frametest fullbench
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
valgrind --leak-check=yes ./lz4 -9 -BD -f tmp $(VOID)
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID)
./datagen -g16KB -s2 > tmp2
./datagen -g16KB -s3 > tmp3
valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3
./datagen -g16MB > tmp
valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp tmp2
valgrind --leak-check=yes ./lz4 -t tmp2
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
./datagen -g256MB > tmp
valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp $(VOID)
valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID)
rm tmp*
valgrind --leak-check=yes ./fuzzer -i64 -t1
valgrind --leak-check=yes ./frametest -i256
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...

31
programs/bench.c
View File

@ -58,9 +58,9 @@
#include "lz4.h"
#define COMPRESSOR0 LZ4_compress_local
static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); }
static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); }
#include "lz4hc.h"
#define COMPRESSOR1 LZ4_compressHC2
#define COMPRESSOR1 LZ4_compress_HC
#define DEFAULTCOMPRESSOR COMPRESSOR0
#include "xxhash.h"
@ -121,8 +121,8 @@ struct chunkParameters
struct compressionParameters
{
int (*compressionFunction)(const char*, char*, int, int);
int (*decompressionFunction)(const char*, char*, int);
int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
int (*decompressionFunction)(const char* src, char* dst, int dstSize);
};
@ -281,17 +281,13 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
/* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
if (inFile==NULL)
{
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
/* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; }
if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; }
if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
{
@ -306,7 +302,6 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
compressedBuffSize = nbChunks * maxCompressedChunkSize;
compressedBuffer = (char*)malloc((size_t)compressedBuffSize);
if (!orig_buff || !compressedBuffer)
{
DISPLAY("\nError: not enough memory!\n");
@ -376,11 +371,12 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel);
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
nbLoops += !nbLoops; /* avoid division by zero */
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
@ -402,8 +398,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
}
milliTime = BMK_GetMilliSpan(milliTime);
nbLoops += !nbLoops; /* avoid division by zero */
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
@ -413,9 +410,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
if (crcOrig==crcCheck)
{
if (ratio<100.)
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
else
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
}
totals += benchedSize;
totalz += cSize;
@ -431,7 +428,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
if (nbFiles > 1)
DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); }
if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); }
return 0;
}

152
programs/frametest.c
View File

@ -238,10 +238,11 @@ int basicTests(U32 seed, double compressibility)
U32 randState = seed;
size_t cSize, testSize;
LZ4F_preferences_t prefs;
LZ4F_decompressionContext_t dCtx;
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
// Create compressible test buffer
/* Create compressible test buffer */
memset(&prefs, 0, sizeof(prefs));
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
@ -249,7 +250,7 @@ int basicTests(U32 seed, double compressibility)
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
// Trivial tests : one-step frame
/* Trivial tests : one-step frame */
testSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "Using NULL preferences : \n");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
@ -276,12 +277,58 @@ int basicTests(U32 seed, double compressibility)
if (crcDest != crcOrig) goto _output_error;
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
DISPLAYLEVEL(4, "Reusing decompression context \n");
{
size_t iSize = compressedBufferSize - 4;
const BYTE* cBuff = (const BYTE*) compressedBuffer;
DISPLAYLEVEL(3, "Missing last 4 bytes : ");
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
if (!errorCode) goto _output_error;
DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode);
cBuff += iSize;
iSize = errorCode;
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL);
if (errorCode != 0) goto _output_error;
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error;
}
{
size_t oSize = 0;
size_t iSize = 0;
LZ4F_frameInfo_t fi;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
DISPLAYLEVEL(3, "get FrameInfo on null input : ");
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
iSize = 6;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
ip += iSize;
DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
iSize = 15 - iSize;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, " correctly decoded \n");
ip += iSize;
}
DISPLAYLEVEL(3, "Byte after byte : \n");
while (ip < iend)
{
size_t oSize = oend-op;
size_t iSize = 1;
//DISPLAY("%7i \n", (int)(ip-(BYTE*)compressedBuffer));
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
@ -289,28 +336,28 @@ int basicTests(U32 seed, double compressibility)
}
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error;
DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH);
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
}
DISPLAYLEVEL(3, "Using 64 KB block : \n");
prefs.frameInfo.blockSizeID = max64KB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 256 KB block : \n");
prefs.frameInfo.blockSizeID = max256KB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max256KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@ -350,33 +397,33 @@ int basicTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 1 MB block : \n");
prefs.frameInfo.blockSizeID = max1MB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max1MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 4 MB block : \n");
prefs.frameInfo.blockSizeID = max4MB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
prefs.frameInfo.blockSizeID = LZ4F_max4MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@ -385,7 +432,6 @@ int basicTests(U32 seed, double compressibility)
size_t errorCode;
BYTE* const ostart = (BYTE*)compressedBuffer;
BYTE* op = ostart;
LZ4F_compressionContext_t cctx;
errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
@ -430,6 +476,7 @@ int basicTests(U32 seed, double compressibility)
errorCode = LZ4F_freeCompressionContext(cctx);
if (LZ4F_isError(errorCode)) goto _output_error;
cctx = NULL;
}
DISPLAYLEVEL(3, "Skippable frame test : \n");
@ -500,10 +547,6 @@ int basicTests(U32 seed, double compressibility)
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
/* release memory */
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
}
DISPLAY("Basic tests completed \n");
@ -511,6 +554,8 @@ _end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
LZ4F_freeCompressionContext(cctx); cctx = NULL;
return testResult;
_output_error:
@ -523,8 +568,8 @@ _output_error:
static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
{
int p=0;
BYTE* b1=(BYTE*)buff1;
BYTE* b2=(BYTE*)buff2;
const BYTE* b1=(const BYTE*)buff1;
const BYTE* b2=(const BYTE*)buff2;
if (nonContiguous)
{
DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
@ -537,7 +582,7 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility)
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
unsigned testResult = 0;
unsigned testNb = 0;
@ -548,10 +593,15 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cCtx = NULL;
size_t result;
const U32 startTime = FUZ_GetMilliStart();
XXH64_state_t xxh64;
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
/* Init */
duration *= 1000;
/* Create buffers */
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
@ -566,10 +616,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
/* jump to requested testNb */
for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
/* main fuzzer test loop */
for ( ; testNb < nbTests; testNb++)
for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
{
U32 randState = coreRand ^ prime1;
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
@ -591,9 +641,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
memset(&prefs, 0, sizeof(prefs));
memset(&cOptions, 0, sizeof(cOptions));
memset(&dOptions, 0, sizeof(dOptions));
prefs.frameInfo.blockMode = (blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
prefs.frameInfo.contentSize = frameContentSize;
prefs.autoFlush = autoflush;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
@ -662,6 +712,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */
XXH64_reset(&xxh64, 1);
if (maxBits < 3) maxBits = 3;
while (ip < iend)
{
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
@ -673,7 +724,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
dOptions.stableDst = FUZ_rand(&randState) & 1;
if (nonContiguousDst==2) dOptions.stableDst = 0;
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
XXH64_update(&xxh64, op, (U32)oSize);
totalOut += oSize;
@ -704,7 +756,7 @@ _end:
if (pause)
{
DISPLAY("press enter to finish \n");
getchar();
(void)getchar();
}
return testResult;
@ -721,6 +773,7 @@ int FUZ_usage(void)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
@ -739,6 +792,7 @@ int main(int argc, char** argv)
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 duration=0;
/* Check command line */
programName = argv[0];
@ -746,9 +800,9 @@ int main(int argc, char** argv)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
if(!argument) continue; /* Protection if argument empty */
// Decode command (note : aggregated commands are allowed)
/* Decode command (note : aggregated commands are allowed) */
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt"))
@ -781,7 +835,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
nbTests=0;
nbTests=0; duration=0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
@ -789,6 +843,32 @@ int main(int argc, char** argv)
argument++;
}
break;
case 'T':
argument++;
nbTests = 0; duration = 0;
for (;;)
{
switch(*argument)
{
case 'm': duration *= 60; argument++; continue;
case 's':
case 'n': argument++; continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': duration *= 10; duration += *argument++ - '0'; continue;
}
break;
}
break;
case 's':
argument++;
seed=0;
@ -841,5 +921,5 @@ int main(int argc, char** argv)
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
if (result) return 1;
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
}

522
programs/fullbench.c
View File

@ -30,7 +30,7 @@
#define _CRT_SECURE_NO_WARNINGS
#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
// Unix Large Files support (>4GB)
/* Unix Large Files support (>4GB) */
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
# define _LARGEFILE_SOURCE
# define _FILE_OFFSET_BITS 64
@ -47,17 +47,17 @@
/**************************************
* Includes
**************************************/
#include <stdlib.h> // malloc
#include <stdio.h> // fprintf, fopen, ftello64
#include <sys/types.h> // stat64
#include <sys/stat.h> // stat64
#include <string.h> // strcmp
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include <string.h> /* strcmp */
// Use ftime() if gettimeofday() is not available on your target
/* Use ftime() if gettimeofday() is not available on your target */
#if defined(BMK_LEGACY_TIMER)
# include <sys/timeb.h> // timeb, ftime
# include <sys/timeb.h> /* timeb, ftime */
#else
# include <sys/time.h> // gettimeofday
# include <sys/time.h> /* gettimeofday */
#endif
#include "lz4.h"
@ -75,16 +75,11 @@
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
// GCC does not support _rotl outside of Windows
#if !defined(_WIN32)
# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
#endif
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@ -118,7 +113,7 @@
#define GB *(1U<<30)
#define KNUTH 2654435761U
#define MAX_MEM (1984 MB)
#define MAX_MEM (1920 MB)
#define DEFAULT_CHUNKSIZE (4 MB)
#define ALL_COMPRESSORS 0
@ -139,53 +134,53 @@ struct chunkParameters
/**************************************
* MACRO
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__)
#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__)
/**************************************
* Benchmark Parameters
**************************************/
static int g_chunkSize = DEFAULT_CHUNKSIZE;
static int g_nbIterations = NBLOOPS;
static int g_pause = 0;
static int g_compressionTest = 1;
static int g_compressionAlgo = ALL_COMPRESSORS;
static int g_decompressionTest = 1;
static int g_decompressionAlgo = ALL_DECOMPRESSORS;
static int g_noPrompt = 0;
//**************************************
// Benchmark Parameters
//**************************************
static int chunkSize = DEFAULT_CHUNKSIZE;
static int nbIterations = NBLOOPS;
static int BMK_pause = 0;
static int compressionTest = 1;
static int decompressionTest = 1;
static int compressionAlgo = ALL_COMPRESSORS;
static int decompressionAlgo = ALL_DECOMPRESSORS;
static int no_prompt = 0;
void BMK_SetBlocksize(int bsize)
static void BMK_setBlocksize(int bsize)
{
chunkSize = bsize;
DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
g_chunkSize = bsize;
DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10);
}
void BMK_SetNbIterations(int nbLoops)
static void BMK_setNbIterations(int nbLoops)
{
nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", nbIterations);
g_nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", g_nbIterations);
}
void BMK_SetPause(void)
static void BMK_setPause(void)
{
BMK_pause = 1;
g_pause = 1;
}
//*********************************************************
// Private functions
//*********************************************************
/*********************************************************
* Private functions
*********************************************************/
#if defined(BMK_LEGACY_TIMER)
static int BMK_GetMilliStart(void)
{
// Based on Legacy ftime()
// Rolls over every ~ 12.1 days (0x100000/24/60/60)
// Use GetMilliSpan to correct for rollover
/* Based on Legacy ftime()
* Rolls over every ~ 12.1 days (0x100000/24/60/60)
* Use GetMilliSpan to correct for rollover */
struct timeb tb;
int nCount;
ftime( &tb );
@ -197,8 +192,8 @@ static int BMK_GetMilliStart(void)
static int BMK_GetMilliStart(void)
{
// Based on newer gettimeofday()
// Use GetMilliSpan to correct for rollover
/* Based on newer gettimeofday()
* Use GetMilliSpan to correct for rollover */
struct timeval tv;
int nCount;
gettimeofday(&tv, NULL);
@ -253,7 +248,7 @@ static U64 BMK_GetFileSize(char* infilename)
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
@ -381,109 +376,144 @@ start:
#endif // __SSSE3__
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
static LZ4_stream_t LZ4_stream;
static void local_LZ4_resetDictT(void)
{
return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
LZ4_resetStream(&LZ4_stream);
}
static void* stateLZ4;
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
static void local_LZ4_createStream(void)
{
return LZ4_compress_withState(stateLZ4, in, out, inSize);
}
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
}
static LZ4_stream_t* ctx;
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_continue(ctx, in, out, inSize);
}
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
}
LZ4_stream_t LZ4_dict;
static void* local_LZ4_resetDictT(const char* fake)
{
(void)fake;
memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
return NULL;
}
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
{
return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
}
static void* stateLZ4HC;
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
}
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize);
}
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
{
return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL);
LZ4_resetStream(&LZ4_stream);
}
static int local_LZ4_saveDict(const char* in, char* out, int inSize)
{
(void)in;
return LZ4_saveDict(&LZ4_dict, out, inSize);
return LZ4_saveDict(&LZ4_stream, out, inSize);
}
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compress_default_large(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compress_default_small(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compress_fast0(const char* in, char* out, int inSize)
{
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0);
}
static int local_LZ4_compress_fast1(const char* in, char* out, int inSize)
{
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1);
}
static int local_LZ4_compress_fast2(const char* in, char* out, int inSize)
{
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2);
}
static int local_LZ4_compress_fast17(const char* in, char* out, int inSize)
{
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17);
}
static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
}
static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
}
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_withState(&LZ4_stream, in, out, inSize);
}
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_withState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_continue(&LZ4_stream, in, out, inSize);
}
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_limitedOutput_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1);
}
/* declare hidden function */
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
{
return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize);
}
/* HC compression functions */
LZ4_streamHC_t LZ4_streamHC;
static void local_LZ4_resetStreamHC(void)
{
LZ4_resetStreamHC(&LZ4_streamHC, 0);
}
LZ4_streamHC_t LZ4_dictHC;
static int local_LZ4_saveDictHC(const char* in, char* out, int inSize)
{
(void)in;
return LZ4_saveDictHC(&LZ4_dictHC, out, inSize);
return LZ4_saveDictHC(&LZ4_streamHC, out, inSize);
}
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compressHC_withStateHC(&LZ4_streamHC, in, out, inSize);
}
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_withStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_continue(&LZ4_streamHC, in, out, inSize);
}
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compressHC_limitedOutput_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1);
}
/* decompression functions */
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
//lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize);
LZ4_decompress_fast(in, out, outSize);
return outSize;
}
static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_fast_withPrefix64k(in, out, outSize);
return outSize;
}
static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
@ -512,6 +542,13 @@ static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSi
return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
}
/* frame functions */
static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
{
return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL);
}
static LZ4F_decompressionContext_t g_dCtx;
static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize)
@ -526,83 +563,65 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS
}
#define NB_COMPRESSION_ALGORITHMS 100
#define NB_DECOMPRESSION_ALGORITHMS 100
int fullSpeedBench(char** fileNamesTable, int nbFiles)
{
int fileIdx=0;
char* orig_buff;
# define NB_COMPRESSION_ALGORITHMS 16
double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
# define NB_DECOMPRESSION_ALGORITHMS 9
double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
size_t errorCode;
/* Init */
errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode))
{
DISPLAY("dctx allocation issue \n");
return 10;
}
if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; }
// Loop for each file
/* Loop for each fileName */
while (fileIdx<nbFiles)
{
FILE* inFile;
char* orig_buff = NULL;
struct chunkParameters* chunkP = NULL;
char* compressed_buff=NULL;
char* inFileName;
U64 inFileSize;
size_t benchedSize;
int nbChunks;
int maxCompressedChunkSize;
struct chunkParameters* chunkP;
size_t readSize;
char* compressed_buff; int compressedBuffSize;
int compressedBuffSize;
U32 crcOriginal;
// Init
stateLZ4 = LZ4_createStream();
stateLZ4HC = LZ4_createStreamHC();
// Check file existence
/* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
if (inFile==NULL)
{
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
// Memory allocation & restrictions
/* Memory size adjustments */
inFileSize = BMK_GetFileSize(inFileName);
if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; }
benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;
if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; }
if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
benchedSize = (size_t) BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */
if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
{
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
}
// Alloc
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
orig_buff = (char*) malloc((size_t)benchedSize);
nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
maxCompressedChunkSize = LZ4_compressBound(chunkSize);
/* Allocation */
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters));
orig_buff = (char*) malloc(benchedSize);
nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize);
maxCompressedChunkSize = LZ4_compressBound(g_chunkSize);
compressedBuffSize = nbChunks * maxCompressedChunkSize;
compressed_buff = (char*)malloc((size_t)compressedBuffSize);
if(!orig_buff || !compressed_buff)
if(!chunkP || !orig_buff || !compressed_buff)
{
DISPLAY("\nError: not enough memory!\n");
free(orig_buff);
free(compressed_buff);
free(chunkP);
fclose(inFile);
return 12;
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
free(orig_buff);
free(compressed_buff);
free(chunkP);
return(12);
}
// Fill input buffer
/* Fill in src buffer */
DISPLAY("Loading %s... \r", inFileName);
readSize = fread(orig_buff, 1, benchedSize, inFile);
fclose(inFile);
@ -616,11 +635,11 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
return 13;
}
// Calculating input Checksum
crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0);
/* Calculating input Checksum */
crcOriginal = XXH32(orig_buff, benchedSize,0);
// Bench
/* Bench */
{
int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
size_t cSize=0;
@ -629,67 +648,80 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName);
// Compression Algorithms
for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
/* Bench Compression Algorithms */
for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++)
{
const char* compressorName;
int (*compressionFunction)(const char*, char*, int);
void* (*initFunction)(const char*) = NULL;
void (*initFunction)(void) = NULL;
double bestTime = 100000000.;
// Init data chunks
/* filter compressionAlgo only */
if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue;
/* Init data chunks */
{
int i;
size_t remaining = benchedSize;
char* in = orig_buff;
char* out = compressed_buff;
nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
for (i=0; i<nbChunks; i++)
{
chunkP[i].id = i;
chunkP[i].origBuffer = in; in += chunkSize;
if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].origBuffer = in; in += g_chunkSize;
if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
}
}
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
switch(cAlgNb)
{
case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
case 0 : DISPLAY("Compression functions : \n"); continue;
case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break;
case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break;
case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
case 10: compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break;
case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
chunkP[0].origSize = (int)benchedSize; nbChunks=1;
break;
case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict";
LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize);
case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict";
LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize);
break;
case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC";
LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize);
case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC";
LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize);
break;
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
case 60: DISPLAY("Obsolete compression functions : \n"); continue;
case 61: compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break;
case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break;
default :
continue; /* unknown ID : just skip */
}
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++)
{
double averageTime;
int milliTime;
PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize);
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warming up memory
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */
nb_loops = 0;
milliTime = BMK_GetMilliStart();
@ -697,17 +729,17 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
milliTime = BMK_GetMilliStart();
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer);
if (initFunction!=NULL) initFunction();
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
}
if (initFunction!=NULL) free(ctx);
nb_loops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
nb_loops += !nb_loops; /* avoid division by zero */
averageTime = (double)milliTime / nb_loops;
if (averageTime < bestTime) bestTime = averageTime;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
@ -719,24 +751,22 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
else
DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
totalCTime[cAlgNb] += bestTime;
totalCSize[cAlgNb] += cSize;
}
// Prepare layout for decompression
// Init data chunks
/* Prepare layout for decompression */
/* Init data chunks */
{
int i;
size_t remaining = benchedSize;
char* in = orig_buff;
char* out = compressed_buff;
nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize);
nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
for (i=0; i<nbChunks; i++)
{
chunkP[i].id = i;
chunkP[i].origBuffer = in; in += chunkSize;
if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].origBuffer = in; in += g_chunkSize;
if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
}
@ -747,39 +777,45 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
}
// Decompression Algorithms
for (dAlgNb=1; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)
/* Decompression Algorithms */
for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++)
{
//const char* dName = decompressionNames[dAlgNb];
const char* dName;
int (*decompressionFunction)(const char*, char*, int, int);
double bestTime = 100000000.;
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;
if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue;
switch(dAlgNb)
{
case 0: DISPLAY("Decompression functions : \n"); continue;
case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break;
case 2: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; dName = "LZ4_decompress_fast_withPrefix64k"; break;
case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break;
case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break;
case 5: decompressionFunction = LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break;
case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress";
errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL);
if (LZ4F_isError(errorCode)) { DISPLAY("Preparation error compressing frame\n"); return 1; }
if (LZ4F_isError(errorCode))
{
DISPLAY("Error while preparing compressed frame\n");
free(orig_buff);
free(compressed_buff);
free(chunkP);
return 1;
}
chunkP[0].origSize = (int)benchedSize;
chunkP[0].compressedSize = (int)errorCode;
nbChunks = 1;
break;
default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
default :
continue; /* skip if unknown ID */
}
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++)
{
double averageTime;
int milliTime;
@ -802,6 +838,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
}
milliTime = BMK_GetMilliSpan(milliTime);
nb_loops += !nb_loops; /* Avoid division by zero */
averageTime = (double)milliTime / nb_loops;
if (averageTime < bestTime) bestTime = averageTime;
@ -813,24 +850,21 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
}
DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
totalDTime[dAlgNb] += bestTime;
}
}
free(orig_buff);
free(compressed_buff);
free(chunkP);
}
if (BMK_pause) { printf("press enter...\n"); getchar(); }
LZ4F_freeDecompressionContext(g_dCtx);
if (g_pause) { printf("press enter...\n"); (void)getchar(); }
return 0;
}
int usage(char* exename)
static int usage(char* exename)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
@ -841,7 +875,7 @@ int usage(char* exename)
return 0;
}
int usage_advanced(void)
static int usage_advanced(void)
{
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS);
@ -851,7 +885,7 @@ int usage_advanced(void)
return 0;
}
int badusage(char* exename)
static int badusage(char* exename)
{
DISPLAY("Wrong parameters\n");
usage(exename);
@ -877,7 +911,7 @@ int main(int argc, char** argv)
if(!argument) continue; // Protection if argument empty
if (!strcmp(argument, "--no-prompt"))
{
no_prompt = 1;
g_noPrompt = 1;
continue;
}
@ -892,22 +926,22 @@ int main(int argc, char** argv)
{
// Select compression algorithm only
case 'c':
decompressionTest = 0;
g_decompressionTest = 0;
while ((argument[1]>= '0') && (argument[1]<= '9'))
{
compressionAlgo *= 10;
compressionAlgo += argument[1] - '0';
g_compressionAlgo *= 10;
g_compressionAlgo += argument[1] - '0';
argument++;
}
break;
// Select decompression algorithm only
case 'd':
compressionTest = 0;
g_compressionTest = 0;
while ((argument[1]>= '0') && (argument[1]<= '9'))
{
decompressionAlgo *= 10;
decompressionAlgo += argument[1] - '0';
g_decompressionAlgo *= 10;
g_decompressionAlgo += argument[1] - '0';
argument++;
}
break;
@ -928,7 +962,7 @@ int main(int argc, char** argv)
{
int B = argument[1] - '0';
int S = 1 << (8 + 2*B);
BMK_SetBlocksize(S);
BMK_setBlocksize(S);
argument++;
break;
}
@ -940,16 +974,16 @@ _exit_blockProperties:
// Modify Nb Iterations
case 'i':
if ((argument[1] >='1') && (argument[1] <='9'))
if ((argument[1] >='0') && (argument[1] <='9'))
{
int iters = argument[1] - '0';
BMK_SetNbIterations(iters);
BMK_setNbIterations(iters);
argument++;
}
break;
// Pause at the end (hidden option)
case 'p': BMK_SetPause(); break;
case 'p': BMK_setPause(); break;
// Unknown command
default : badusage(exename); return 1;

269
programs/fuzzer.c
View File

@ -19,7 +19,6 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : http://code.google.com/p/lz4
- LZ4 source mirror : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
@ -154,10 +153,10 @@ static U32 FUZ_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 ^= PRIME2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
return rand32 >> 3;
return rand32;
}
@ -169,27 +168,28 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
size_t pos = 0;
U32 P32 = (U32)(32768 * proba);
// First Byte
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
/* First Bytes */
while (pos < 20)
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
// Select : Literal (noise) or copy (within 64K)
/* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32)
{
// Copy (within 64K)
/* Copy (within 64K) */
size_t match, d;
size_t length = FUZ_RANDLENGTH + 4;
size_t offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos;
while (offset > pos) offset >>= 1;
d = pos + length;
if (d > bufferSize) d = bufferSize;
while (d > bufferSize) d = bufferSize;
match = pos - offset;
while (pos < d) BBuffer[pos++] = BBuffer[match++];
}
else
{
// Literal (noise)
/* Literal (noise) */
size_t d;
size_t length = FUZ_RANDLENGTH;
d = pos + length;
@ -210,7 +210,7 @@ static int FUZ_AddressOverflow(void)
printf("Overflow tests : ");
// Only possible in 32-bits
/* Only possible in 32-bits */
if (sizeof(void*)==8)
{
printf("64 bits mode : no overflow \n");
@ -300,7 +300,17 @@ static void FUZ_displayUpdate(unsigned testNb)
}
static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility)
static void FUZ_findDiff(const void* buff1, const void* buff2)
{
const BYTE* b1 = (const BYTE*)buff1;
const BYTE* b2 = (const BYTE*)buff2;
size_t i=0;
while (b1[i]==b2[i]) i++;
DISPLAY("Wrong Byte at position %u\n", (unsigned)i);
}
static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration)
{
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
@ -313,34 +323,35 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
int ret;
unsigned cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
LZ4_streamHC_t LZ4dictHC;
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
int result = 0;
const U32 startTime = FUZ_GetMilliStart();
// init
/* init */
memset(&LZ4dict, 0, sizeof(LZ4dict));
duration *= 1000;
// Create compressible test buffer
/* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// move to startCycle
/* move to startCycle */
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
(void)FUZ_rand(&coreRandState);
if (0) // some problems related to dictionary re-use; in this case, enable this loop
if (0) /* some problems are related to dictionary re-use; in this case, enable this loop */
{
int dictSize, blockSize, blockStart;
char* dict;
@ -362,8 +373,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
/* Main test loop */
for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetMilliSpan(startTime) < duration) ; cycleNb++)
{
U32 testNb = 0;
char* dict;
@ -375,7 +386,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
(void)FUZ_rand(&coreRandState);
randState = coreRandState ^ PRIME3;
// Select block to test
/* Select block to test */
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
@ -385,23 +396,58 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
/* Compression tests */
// Test compression HC
/* Test compression destSize */
FUZ_DISPLAYTEST;
{
int srcSize = blockSize;
int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
char endCheck = FUZ_rand(&randState) & 255;
compressedBuffer[targetSize] = endCheck;
ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !");
DISPLAY("destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
if (targetSize>0)
{
FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
/* check correctness */
FUZ_DISPLAYTEST;
crcOrig = XXH32(block, srcSize, 0);
compressedSize = ret;
endCheck = FUZ_rand(&randState) & 255;
decodedBuffer[srcSize] = endCheck;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !");
crcCheck = XXH32(decodedBuffer, srcSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data");
DISPLAY(" OK \n");
}
else
DISPLAY(" \n");
}
/* Test compression HC */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
/* Test compression HC using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
/* Test compression using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
/* Test compression */
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
@ -411,7 +457,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
/* Test decoding with output size being exactly what's necessary => must work */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
@ -419,19 +465,19 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
/* Test decoding with one byte missing => must fail */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
/* Test decoding with one byte too much => must fail */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
/* Test decoding with output size exactly what's necessary => must work */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
@ -545,16 +591,18 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
/* Compress using dictionary */
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
free (LZ4continue);
{
LZ4_stream_t LZ4_stream;
LZ4_resetStream(&LZ4_stream);
LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */
blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
}
/* Decompress with dictionary as prefix */
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
@ -567,14 +615,14 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
// Compress using External dictionary
/* Compress using External dictionary */
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict
dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
@ -583,7 +631,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
@ -591,7 +639,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
/* Decompress with dictionary as external */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
@ -599,11 +647,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
@ -617,7 +661,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
@ -638,9 +682,9 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
}
// Compress HC using External dictionary
/* Compress HC using External dictionary */
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 7); // even bigger separation
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
@ -665,29 +709,26 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
// ***** End of tests *** //
// Fill stats
/* ***** End of tests *** */
/* Fill stats */
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
if (nbCycles<=1) nbCycles = cycleNb; /* end by time */
bytes += !bytes; /* avoid division by 0 */
printf("\r%7u /%7u - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
/* release memory */
{
_exit:
free(CNBuffer);
@ -719,13 +760,13 @@ static void FUZ_unitTests(void)
char ringBuffer[ringBufferSize];
U32 randState = 1;
// Init
/* Init */
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
// 32-bits address space overflow test
/* 32-bits address space overflow test */
FUZ_AddressOverflow();
// LZ4 streaming tests
/* LZ4 streaming tests */
{
LZ4_stream_t* statePtr;
LZ4_stream_t streamingState;
@ -733,12 +774,12 @@ static void FUZ_unitTests(void)
U64 crcNew;
int result;
// Allocation test
/* Allocation test */
statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr);
// simple compression test
/* simple compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState);
result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@ -749,7 +790,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// ring buffer test
/* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
@ -783,7 +824,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// prepare next message
/* prepare next message */
iNext += messageSize;
rNext += messageSize;
dNext += messageSize;
@ -794,21 +835,20 @@ static void FUZ_unitTests(void)
}
}
// LZ4 HC streaming tests
/* LZ4 HC streaming tests */
{
LZ4_streamHC_t* sp;
LZ4_streamHC_t sHC;
//XXH64_state_t xxh;
U64 crcOrig;
U64 crcNew;
int result;
// Allocation test
/* Allocation test */
sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp);
// simple compression test
/* simple HC compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
@ -819,7 +859,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
// simple dictionary compression test
/* simple dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
@ -831,7 +871,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
// multiple HC compression test with dictionary
/* multiple HC compression test with dictionary */
{
int result1, result2;
int segSize = testCompressedSize / 2;
@ -851,7 +891,7 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
}
// remote dictionary HC compression test
/* remote dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
@ -863,7 +903,7 @@ static void FUZ_unitTests(void)
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
// multiple HC compression with ext. dictionary
/* multiple HC compression with ext. dictionary */
{
XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
@ -912,7 +952,7 @@ static void FUZ_unitTests(void)
}
}
// ring buffer test
/* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
@ -956,24 +996,51 @@ static void FUZ_unitTests(void)
}
}
// small decoder-side ring buffer test
/* small decoder-side ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState;
const U32 maxMessageSizeLog = 10;
const U32 maxMessageSizeLog = 12;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
U32 messageSize;
U32 totalMessageSize = 0;
U32 iNext = 0;
U32 dNext = 0;
const U32 dBufferSize = 64 KB + maxMessageSizeMask;
const U32 dBufferSize = 64 KB;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_setStreamDecode(&decodeState, NULL, 0);
#define BSIZE1 65537
#define BSIZE2 16435
/* first block */
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
while (totalMessageSize < 9 MB)
{
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@ -983,18 +1050,20 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
if (crcOrig != crcNew)
FUZ_findDiff(testInput + iNext, testVerify + dNext);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
iNext = (FUZ_rand(&randState) & 65535);
if (dNext + messageSize > dBufferSize) dNext = 0;
if (dNext > dBufferSize) dNext = 0;
}
}
}
@ -1013,6 +1082,7 @@ static int FUZ_usage(char* programName)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
@ -1033,8 +1103,9 @@ int main(int argc, char** argv)
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int pause = 0;
char* programName = argv[0];
U32 duration = 0;
// Check command line
/* Check command line */
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
@ -1066,7 +1137,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
nbTests=0;
nbTests = 0; duration = 0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
@ -1075,6 +1146,31 @@ int main(int argc, char** argv)
}
break;
case 'T':
argument++;
nbTests = 0; duration = 0;
for (;;)
{
switch(*argument)
{
case 'm': duration *= 60; argument++; continue;
case 's':
case 'n': argument++; continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': duration *= 10; duration += *argument++ - '0'; continue;
}
break;
}
break;
case 's':
argument++;
seed=0; seedset=1;
@ -1115,7 +1211,6 @@ int main(int argc, char** argv)
}
}
// Get Seed
printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
@ -1127,11 +1222,11 @@ int main(int argc, char** argv)
if (nbTests<=0) nbTests=1;
{
int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
if (pause)
{
DISPLAY("press enter ... \n");
getchar();
(void)getchar();
}
return result;
}

14
programs/lz4.1
View File

@ -158,13 +158,13 @@ for files that have not been compressed with
force write to standard output, even if it is the console
.TP
.BR \-m
.BR \-m ", " \--multiple
Multiple file names.
By default, the second filename is used as the output filename for the compressed file.
With
.B -m
, you can specify any number of input filenames, each of them will be compressed
with the resulting compressed file named
, you can specify any number of input filenames. Each of them will be compressed
independently, and the resulting name of the compressed file will be
.B filename.lz4
.
@ -177,15 +177,15 @@ with the resulting compressed file named
block dependency (improve compression ratio)
.TP
.B \--[no-]frame-crc
disable stream checksum (default:enabled)
select frame checksum (default:enabled)
.TP
.B \--[no-]content-size
compressed file includes original size (default:not present)
header includes original size (default:not present)
Note : this option can only be activated when the original size can be determined,
hence for a file. It won't work with unknown source size, such as stdin pipe.
hence for a file. It won't work with unknown source size, such as stdin or pipe.
.TP
.B \--[no-]sparse
enable sparse file (default:disabled)(experimental)
sparse file support (default:enabled)
.TP
.B \-l
use Legacy format (useful for Linux Kernel compression)

72
programs/lz4cli.c
View File

@ -65,7 +65,7 @@
/****************************
* OS-specific Includes
*****************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <io.h> /* _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */
@ -102,7 +102,7 @@
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */
/**************************************
@ -175,7 +175,7 @@ static int usage_advanced(void)
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
DISPLAY( "--sparse : enable sparse file (default:disabled)(experimental)\n");
DISPLAY( "--[no-]sparse : sparse file support (default:enabled)\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b : benchmark file(s)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
@ -251,7 +251,7 @@ static int badusage(void)
static void waitEnter(void)
{
DISPLAY("Press enter to continue...\n");
getchar();
(void)getchar();
}
@ -265,7 +265,8 @@ int main(int argc, char** argv)
forceStdout=0,
forceCompress=0,
main_pause=0,
multiple_inputs=0;
multiple_inputs=0,
operationResult=0;
const char* input_filename=0;
const char* output_filename=0;
char* dynNameSpace=0;
@ -294,12 +295,13 @@ int main(int argc, char** argv)
/* long commands (--long-word) */
if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; }
if ((!strcmp(argument, "--decompress"))
|| (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
|| (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
@ -311,6 +313,7 @@ int main(int argc, char** argv)
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */
/* Short commands (note : aggregated short commands are allowed) */
if (argument[0]=='-')
{
@ -420,11 +423,15 @@ int main(int argc, char** argv)
/* Modify Nb Iterations (benchmark only) */
case 'i':
if ((argument[1] >='1') && (argument[1] <='9'))
{
int iters = argument[1] - '0';
unsigned iters = 0;
while ((argument[1] >='0') && (argument[1] <='9'))
{
iters *= 10;
iters += argument[1] - '0';
argument++;
}
BMK_setNbIterations(iters);
argument++;
}
break;
@ -463,14 +470,19 @@ int main(int argc, char** argv)
if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
/* No input filename ==> use stdin */
if (multiple_inputs) input_filename = inFileNames[0], output_filename = (char*)(inFileNames[0]);
if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]);
if(!input_filename) { input_filename=stdinmark; }
/* Check if input or output are defined as console; trigger an error in this case */
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
/* Check if benchmark is selected */
if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel);
if (bench)
{
int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel);
free((void*)inFileNames);
return bmkResult;
}
/* No output filename ==> try to select one automatically (when possible) */
while (!output_filename)
@ -509,32 +521,40 @@ int main(int argc, char** argv)
/* Check if output is defined as console; trigger an error in this case */
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
/* No warning message in pure pipe mode (stdin + stdout) */
/* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */
if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
LZ4IO_setNotificationLevel(displayLevel);
if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename);
if (decode)
{
if (multiple_inputs)
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION);
else
DEFAULT_DECOMPRESSOR(input_filename, output_filename);
}
else
{
/* compression is default action */
if (legacy_format)
{
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
}
/* compression is default action */
if (legacy_format)
{
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
}
else
{
if (multiple_inputs)
operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
else
{
if (multiple_inputs)
LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
else
DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
}
DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
}
}
if (main_pause) waitEnter();
free(dynNameSpace);
free((void*)inFileNames);
if (operationResult != 0) return operationResult;
return 0;
}

674
programs/lz4io.c
View File

@ -46,22 +46,22 @@
/*****************************
* Includes
*****************************/
#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */
#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include "lz4io.h"
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
#include "lz4frame.h"
/******************************
* OS-specific Includes
******************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
@ -131,7 +131,7 @@ static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT;
static int g_blockChecksum = 0;
static int g_streamChecksum = 1;
static int g_blockIndependence = 1;
static int g_sparseFileSupport = 0;
static int g_sparseFileSupport = 1;
static int g_contentSizeFlag = 0;
static const int minBlockSizeID = 4;
@ -159,7 +159,7 @@ static const int maxBlockSizeID = 7;
#define EXTENDED_ARGUMENTS
#define EXTENDED_HELP
#define EXTENDED_FORMAT
#define DEFAULT_DECOMPRESSOR decodeLZ4S
#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
/* ************************************************** */
@ -240,7 +240,7 @@ static unsigned long long LZ4IO_GetFileSize(const char* infilename)
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* failure, or is not a regular file */
return (unsigned long long)statbuf.st_size;
}
@ -253,7 +253,7 @@ static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))
static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; }
static int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
static int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
{
if (!strcmp (input_filename, stdinmark))
@ -267,6 +267,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam
*pfinput = fopen(input_filename, "rb");
}
if ( *pfinput==0 )
{
DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename);
return 1;
}
if (!strcmp (output_filename, stdoutmark))
{
DISPLAYLEVEL(4,"Using stdout for output\n");
@ -283,18 +289,18 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam
fclose(*pfoutput);
if (!g_overwrite)
{
char ch;
int ch = 'Y';
DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
ch = (char)getchar();
if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
if ((g_displayLevel <= 1) || (*pfinput == stdin))
EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
DISPLAYLEVEL(2, "Overwrite ? (Y/n) : ");
while((ch = getchar()) != '\n' && ch != EOF) /* flush integrated */
if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename);
}
}
*pfoutput = fopen( output_filename, "wb" );
}
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
return 0;
@ -316,16 +322,23 @@ static void LZ4IO_writeLE32 (void* p, unsigned value32)
dstPtr[3] = (unsigned char)(value32 >> 24);
}
static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel)
{
(void)cLevel;
return LZ4_compress_fast(src, dst, srcSize, dstSize, 1);
}
/* LZ4IO_compressFilename_Legacy :
* This function is intentionally "hidden" (not published in .h)
* It generates compressed streams using the old 'legacy' format */
int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel)
{
int (*compressionFunction)(const char*, char*, int);
int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
unsigned long long filesize = 0;
unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
char* in_buff;
char* out_buff;
const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE);
FILE* finput;
FILE* foutput;
clock_t start, end;
@ -334,14 +347,14 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Init */
start = clock();
if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC;
get_fileHandle(input_filename, output_filename, &finput, &foutput);
if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3;
if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput))
EXM_THROW(20, "File error");
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
out_buff = (char*)malloc(outBuffSize);
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
/* Write Archive Header */
@ -359,9 +372,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
filesize += inSize;
/* Compress Block */
outSize = compressionFunction(in_buff, out_buff+4, inSize);
outSize = compressionFunction(in_buff, out_buff+4, inSize, outBuffSize, compressionlevel);
compressedfilesize += outSize+4;
DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
LZ4IO_writeLE32(out_buff, outSize);
@ -372,6 +385,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Status */
end = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
filesize += !filesize; /* avoid divide by zero */
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
{
@ -393,128 +407,210 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
* Compression using Frame format
*********************************************/
int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel)
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
LZ4F_compressionContext_t ctx;
} cRess_t;
static cRess_t LZ4IO_createCResources(void)
{
const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
cRess_t ress;
LZ4F_errorCode_t errorCode;
errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBuffer = malloc(blockSize);
ress.srcBufferSize = blockSize;
ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
return ress;
}
static void LZ4IO_freeCResources(cRess_t ress)
{
LZ4F_errorCode_t errorCode;
free(ress.srcBuffer);
free(ress.dstBuffer);
errorCode = LZ4F_freeCompressionContext(ress.ctx);
if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
}
/*
* LZ4IO_compressFilename_extRess()
* result : 0 : compression completed correctly
* 1 : missing or pb opening srcFileName
*/
static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel)
{
unsigned long long filesize = 0;
unsigned long long compressedfilesize = 0;
char* in_buff;
char* out_buff;
FILE* finput;
FILE* foutput;
clock_t start, end;
int blockSize;
size_t sizeCheck, headerSize, readSize, outBuffSize;
LZ4F_compressionContext_t ctx;
LZ4F_errorCode_t errorCode;
FILE* srcFile;
FILE* dstFile;
void* const srcBuffer = ress.srcBuffer;
void* const dstBuffer = ress.dstBuffer;
const size_t dstBufferSize = ress.dstBufferSize;
const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
size_t sizeCheck, headerSize, readSize;
LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */
LZ4F_preferences_t prefs;
/* Init */
start = clock();
memset(&prefs, 0, sizeof(prefs));
if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3;
errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
get_fileHandle(input_filename, output_filename, &finput, &foutput);
blockSize = LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
/* File check */
if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1;
/* Set compression parameters */
prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel;
prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence;
prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum;
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
if (g_contentSizeFlag)
{
unsigned long long fileSize = LZ4IO_GetFileSize(input_filename);
unsigned long long fileSize = LZ4IO_GetFileSize(srcFileName);
prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */
if (fileSize==0)
DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n");
}
/* Allocate Memory */
in_buff = (char*)malloc(blockSize);
outBuffSize = LZ4F_compressBound(blockSize, &prefs);
out_buff = (char*)malloc(outBuffSize);
if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
/* Write Archive Header */
headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs);
if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header");
compressedfilesize += headerSize;
/* read first block */
readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile);
filesize += readSize;
/* Main Loop */
while (readSize>0)
/* single-block file */
if (readSize < blockSize)
{
size_t outSize;
/* Compress Block */
outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL);
if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Compress in single pass */
size_t cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs);
if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize += cSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
sizeCheck = fwrite(out_buff, 1, outSize, foutput);
if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
filesize += readSize;
sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block");
}
/* End of Stream mark */
headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL);
if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
else
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream");
compressedfilesize += headerSize;
/* multiple-blocks file */
{
/* Write Archive Header */
headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs);
if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header");
compressedfilesize += headerSize;
/* Close & Free */
free(in_buff);
free(out_buff);
fclose(finput);
fclose(foutput);
errorCode = LZ4F_freeCompressionContext(ctx);
if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
/* Main Loop */
while (readSize>0)
{
size_t outSize;
/* Compress Block */
outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
filesize += readSize;
}
/* End of Stream mark */
headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream");
compressedfilesize += headerSize;
}
/* Release files */
fclose (srcFile);
fclose (dstFile);
/* Final Status */
end = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
{
double seconds = (double)(end - start)/CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
}
filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */
return 0;
}
int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel)
{
clock_t start, end;
cRess_t ress;
int issueWithSrcFile = 0;
/* Init */
start = clock();
ress = LZ4IO_createCResources();
/* Compress File */
issueWithSrcFile += LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel);
/* Free resources */
LZ4IO_freeCResources(ress);
/* Final Status */
end = clock();
{
double seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
}
return issueWithSrcFile;
}
#define FNSPACE 30
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel)
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel)
{
int i;
char* outFileName = (char*)malloc(FNSPACE);
int missed_files = 0;
char* dstFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
cRess_t ress;
/* init */
ress = LZ4IO_createCResources();
/* loop on each file */
for (i=0; i<ifntSize; i++)
{
size_t ifnSize = strlen(inFileNamesTable[i]);
if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
strcpy(outFileName, inFileNamesTable[i]);
strcat(outFileName, suffix);
LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel);
if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); }
strcpy(dstFileName, inFileNamesTable[i]);
strcat(dstFileName, suffix);
missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
}
free(outFileName);
return 0;
/* Close & Free */
LZ4IO_freeCResources(ress);
free(dstFileName);
return missed_files;
}
@ -528,15 +624,105 @@ static unsigned LZ4IO_readLE32 (const void* s)
unsigned value32 = srcPtr[0];
value32 += (srcPtr[1]<<8);
value32 += (srcPtr[2]<<16);
value32 += (srcPtr[3]<<24);
value32 += ((unsigned)srcPtr[3])<<24;
return value32;
}
static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
{
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
const size_t* ptrT = bufferT;
size_t bufferSizeT = bufferSize / sizeT;
const size_t* const bufferTEnd = bufferT + bufferSizeT;
static const size_t segmentSizeT = (32 KB) / sizeT;
if (!g_sparseFileSupport) /* normal write */
{
size_t sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) EXM_THROW(68, "Write error : cannot write decoded block");
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB)
{
int seekResult = fseek(file, 1 GB, SEEK_CUR);
if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
while (ptrT < bufferTEnd)
{
size_t seg0SizeT = segmentSizeT;
size_t nb0T;
int seekResult;
/* count leading zeros */
if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
bufferSizeT -= seg0SizeT;
for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
storedSkips += (unsigned)(nb0T * sizeT);
if (nb0T != seg0SizeT) /* not all 0s */
{
size_t sizeCheck;
seekResult = fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
storedSkips = 0;
seg0SizeT -= nb0T;
ptrT += nb0T;
sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
}
ptrT += seg0SizeT;
}
if (bufferSize & maskT) /* size not multiple of sizeT : implies end of block */
{
const char* const restStart = (const char*)bufferTEnd;
const char* restPtr = restStart;
size_t restSize = bufferSize & maskT;
const char* const restEnd = restStart + restSize;
for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd)
{
size_t sizeCheck;
int seekResult = fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(68, "Skip error (end of block)");
storedSkips = 0;
sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
}
}
return storedSkips;
}
static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
{
char lastZeroByte[1] = { 0 };
if (storedSkips>0) /* implies g_sparseFileSupport */
{
int seekResult;
size_t sizeCheck;
storedSkips --;
seekResult = fseek(file, storedSkips, SEEK_CUR);
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
sizeCheck = fwrite(lastZeroByte, 1, 1, file);
if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
}
}
static unsigned g_magicRead = 0;
static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
{
unsigned long long filesize = 0;
char* in_buff;
char* out_buff;
unsigned storedSkips = 0;
/* Allocate Memory */
in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
@ -556,7 +742,7 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
{ /* Cannot read next block : maybe new stream ? */
fseek(finput, -4, SEEK_CUR);
g_magicRead = blockSize;
break;
}
@ -570,10 +756,11 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
filesize += decodeSize;
/* Write Block */
sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n");
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips);
}
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
/* Free */
free(in_buff);
free(out_buff);
@ -582,172 +769,129 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
}
static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
LZ4F_decompressionContext_t dCtx;
} dRess_t;
static const size_t LZ4IO_dBufferSize = 64 KB;
static dRess_t LZ4IO_createDResources(void)
{
unsigned long long filesize = 0;
void* inBuff;
void* outBuff;
# define HEADERMAX 20
char headerBuff[HEADERMAX];
size_t sizeCheck;
const size_t inBuffSize = 256 KB;
const size_t outBuffSize = 256 KB;
LZ4F_decompressionContext_t ctx;
dRess_t ress;
LZ4F_errorCode_t errorCode;
unsigned storedSkips = 0;
/* init */
errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode));
LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */
errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
inBuff = malloc(256 KB);
outBuff = malloc(256 KB);
if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory");
ress.srcBufferSize = LZ4IO_dBufferSize;
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = LZ4IO_dBufferSize;
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
/* Init feed with magic number (already consumed from FILE) */
return ress;
}
static void LZ4IO_freeDResources(dRess_t ress)
{
LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
}
static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile)
{
unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad;
unsigned storedSkips = 0;
/* Init feed with magic number (already consumed from FILE* sFile) */
{
size_t inSize = 4;
size_t outSize=0;
LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER);
errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL);
if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode));
size_t inSize = MAGICNUMBER_SIZE;
size_t outSize= 0;
LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL);
if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
}
/* Main Loop */
for (;;)
for (;nextToLoad;)
{
size_t readSize;
size_t pos = 0;
size_t decodedBytes = ress.dstBufferSize;
/* Read input */
readSize = fread(inBuff, 1, inBuffSize, finput);
if (!readSize) break; /* empty file or stream */
if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize;
readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile);
if (!readSize)
break; /* empty file or stream */
while (pos < readSize)
while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) /* still to read, or still to flush */
{
/* Decode Input (at least partially) */
size_t remaining = readSize - pos;
size_t decodedBytes = outBuffSize;
errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL);
if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode));
decodedBytes = ress.dstBufferSize;
nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
if (!nextToLoad) break;
if (decodedBytes)
{
/* Write Block */
filesize += decodedBytes;
if (g_sparseFileSupport)
{
size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */
size_t* oBuffPosT = oBuffStartT;
size_t oBuffSizeT = decodedBytes / sizeT;
size_t* const oBuffEndT = oBuffStartT + oBuffSizeT;
static const size_t bs0T = (32 KB) / sizeT;
while (oBuffPosT < oBuffEndT)
{
size_t seg0SizeT = bs0T;
size_t nb0T;
int seekResult;
if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT;
oBuffSizeT -= seg0SizeT;
for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ;
storedSkips += (unsigned)(nb0T * sizeT);
if (storedSkips > 1 GB) /* avoid int overflow */
{
seekResult = fseek(foutput, 1 GB, SEEK_CUR);
if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)");
storedSkips -= 1 GB;
}
if (nb0T != seg0SizeT) /* not all 0s */
{
seekResult = fseek(foutput, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
storedSkips = 0;
seg0SizeT -= nb0T;
oBuffPosT += nb0T;
sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput);
if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
}
oBuffPosT += seg0SizeT;
}
if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */
{
const char* const restStart = (char*)oBuffEndT;
const char* restPtr = restStart;
size_t restSize = decodedBytes & maskT;
const char* const restEnd = restStart + restSize;
for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd)
{
int seekResult = fseek(foutput, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(68, "Skip error (end of block)");
storedSkips = 0;
sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput);
if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
}
}
}
else
{
sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block");
}
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips);
}
}
}
if ((g_sparseFileSupport) && (storedSkips>0))
{
int seekResult;
storedSkips --;
seekResult = fseek(foutput, storedSkips, SEEK_CUR);
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
memset(outBuff, 0, 1);
sizeCheck = fwrite(outBuff, 1, 1, foutput);
if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
}
LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
/* Free */
free(inBuff);
free(outBuff);
errorCode = LZ4F_freeDecompressionContext(ctx);
if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
if (nextToLoad!=0)
EXM_THROW(67, "Unfinished stream");
return filesize;
}
static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE])
static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
{
void* buffer = malloc(64 KB);
size_t read = 1, sizeCheck;
unsigned long long total = MAGICNUMBER_SIZE;
unsigned storedSkips = 0;
sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput);
if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start");
sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
while (read)
{
read = fread(buffer, 1, 64 KB, finput);
total += read;
sizeCheck = fwrite(buffer, 1, read, foutput);
if (sizeCheck != read) EXM_THROW(50, "Pass-through error");
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, read, storedSkips);
}
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
free(buffer);
return total;
}
#define ENDOFSTREAM ((unsigned long long)-1)
static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput)
{
unsigned char U32store[MAGICNUMBER_SIZE];
unsigned char MNstore[MAGICNUMBER_SIZE];
unsigned magicNumber, size;
int errorNb;
size_t nbReadBytes;
@ -757,33 +901,41 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
nbCalls++;
/* Check Archive Header */
nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */
if (g_magicRead)
{
magicNumber = g_magicRead;
g_magicRead = 0;
}
else
{
nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
}
if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
switch(magicNumber)
{
case LZ4IO_MAGICNUMBER:
return DEFAULT_DECOMPRESSOR(finput, foutput);
return LZ4IO_decompressLZ4F(ress, finput, foutput);
case LEGACY_MAGICNUMBER:
DISPLAYLEVEL(4, "Detected : Legacy format \n");
return decodeLegacyStream(finput, foutput);
return LZ4IO_decodeLegacyStream(finput, foutput);
case LZ4IO_SKIPPABLE0:
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
nbReadBytes = fread(U32store, 1, 4, finput);
nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
size = LZ4IO_readLE32(U32store); /* Little Endian format */
size = LZ4IO_readLE32(MNstore); /* Little Endian format */
errorNb = fseek(finput, size, SEEK_CUR);
if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
return selectDecoder(finput, foutput);
return selectDecoder(ress, finput, foutput);
EXTENDED_FORMAT;
default:
if (nbCalls == 1) /* just started */
{
if (g_overwrite)
return LZ4IO_passThrough(finput, foutput, U32store);
return LZ4IO_passThrough(finput, foutput, MNstore);
EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */
}
DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
@ -792,43 +944,95 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
}
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename)
{
unsigned long long filesize = 0, decodedSize=0;
FILE* finput;
FILE* foutput;
clock_t start, end;
/* Init */
start = clock();
get_fileHandle(input_filename, output_filename, &finput, &foutput);
if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput))
return 1;
/* sparse file */
if (g_sparseFileSupport && foutput) { SET_SPARSE_FILE_MODE(foutput); }
if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); }
/* Loop over multiple streams */
do
{
decodedSize = selectDecoder(finput, foutput);
decodedSize = selectDecoder(ress, finput, foutput);
if (decodedSize != ENDOFSTREAM)
filesize += decodedSize;
} while (decodedSize != ENDOFSTREAM);
/* Final Status */
end = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
{
double seconds = (double)(end - start)/CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
}
/* Close */
fclose(finput);
fclose(foutput);
/* Error status = OK */
return 0;
}
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
{
dRess_t ress;
clock_t start, end;
int missingFiles = 0;
start = clock();
ress = LZ4IO_createDResources();
missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename);
LZ4IO_freeDResources(ress);
end = clock();
if (end==start) end=start+1;
{
double seconds = (double)(end - start)/CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds);
}
return missingFiles;
}
int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix)
{
int i;
int skippedFiles = 0;
int missingFiles = 0;
char* outFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
char* ifnSuffix = (char*)malloc(suffixSize + 1);
dRess_t ress;
ress = LZ4IO_createDResources();
for (i=0; i<ifntSize; i++)
{
size_t ifnSize = strlen(inFileNamesTable[i]);
strcpy(ifnSuffix, inFileNamesTable[i] + ifnSize - suffixSize);
if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
if (ifnSize <= suffixSize || strcmp(ifnSuffix, suffix) != 0)
{
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;
continue;
}
memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize);
outFileName[ifnSize-suffixSize] = '\0';
missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName);
}
LZ4IO_freeDResources(ress);
free(outFileName);
free(ifnSuffix);
return missingFiles + skippedFiles;
}

3
programs/lz4io.h
View File

@ -51,8 +51,9 @@ static char const nulmark[] = "/dev/null";
int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel);
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix);
/* ************************************************** */
/* ****************** Parameters ******************** */

159
visual/2012/datagen/datagen.vcxproj Normal file
View File

@ -0,0 +1,159 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{D745AE2F-596A-403A-9B91-81A8C6779243}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>datagen</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\datagencli.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\programs\datagen.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

30
visual/2012/datagen/datagen.vcxproj.filters Normal file
View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Fichiers sources">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Fichiers d%27en-tête">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
</Filter>
<Filter Include="Fichiers de ressources">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\programs\datagen.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\programs\datagencli.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\programs\datagen.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
</ItemGroup>
</Project>

166
visual/2012/frametest/frametest.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>frametest</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\lz4.c" />
<ClCompile Include="..\..\..\lib\lz4frame.c" />
<ClCompile Include="..\..\..\lib\lz4hc.c" />
<ClCompile Include="..\..\..\lib\xxhash.c" />
<ClCompile Include="..\..\..\programs\frametest.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\lz4.h" />
<ClInclude Include="..\..\..\lib\lz4frame.h" />
<ClInclude Include="..\..\..\lib\lz4frame_static.h" />
<ClInclude Include="..\..\..\lib\lz4hc.h" />
<ClInclude Include="..\..\..\lib\xxhash.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

51
visual/2012/frametest/frametest.vcxproj.filters Normal file
View File

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Fichiers sources">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Fichiers d%27en-tête">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
</Filter>
<Filter Include="Fichiers de ressources">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\lz4.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\lz4frame.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\lz4hc.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\xxhash.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\programs\frametest.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\lz4.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\lz4frame.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\lz4frame_static.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\lz4hc.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\xxhash.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
</ItemGroup>
</Project>

166
visual/2012/fullbench/fullbench.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>fullbench</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\lz4.c" />
<ClCompile Include="..\..\..\lib\lz4frame.c" />
<ClCompile Include="..\..\..\lib\lz4hc.c" />
<ClCompile Include="..\..\..\lib\xxhash.c" />
<ClCompile Include="..\..\..\programs\fullbench.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\lz4.h" />
<ClInclude Include="..\..\..\lib\lz4frame.h" />
<ClInclude Include="..\..\..\lib\lz4frame_static.h" />
<ClInclude Include="..\..\..\lib\lz4hc.h" />
<ClInclude Include="..\..\..\lib\xxhash.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

51
visual/2012/fullbench/fullbench.vcxproj.filters Normal file
View File

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Fichiers sources">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Fichiers d%27en-tête">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
</Filter>
<Filter Include="Fichiers de ressources">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\lz4.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\lz4frame.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\lz4hc.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\xxhash.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\programs\fullbench.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\lz4.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\lz4frame.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\lz4frame_static.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\lz4hc.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\xxhash.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
</ItemGroup>
</Project>

4
visual/2012/fuzzer/fuzzer.vcxproj
View File

@ -89,6 +89,7 @@
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -102,6 +103,7 @@
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -117,6 +119,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -134,6 +137,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

4
visual/2012/fuzzer/fuzzer.vcxproj.user
View File

@ -1,4 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
</Project>

40
visual/2012/lz4.sln
View File

@ -5,40 +5,60 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Mixed Platforms = Release|Mixed Platforms
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.ActiveCfg = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.Build.0 = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.ActiveCfg = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.Build.0 = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

BIN
visual/2012/lz4.v11.suo
View File

Binary file not shown.

4
visual/2012/lz4/lz4.vcxproj
View File

@ -89,6 +89,7 @@
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -102,6 +103,7 @@
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -117,6 +119,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -134,6 +137,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

4
visual/2012/lz4/lz4.vcxproj.user
View File

@ -1,4 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
</Project>