AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into coverity_scan

Browse Source
This commit is contained in:
Yann Collet 2018-01-13 21:09:02 -08:00
commit 151a50e16e
65 changed files with 5671 additions and 3198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitattributes vendored
View File

@ -19,3 +19,6 @@
# Windows
*.bat text eol=crlf
*.cmd text eol=crlf
# .travis.yml merging
.travis.yml merge=ours

16
INSTALL Normal file
View File

@ -0,0 +1,16 @@
Installation
=============
```
make
make install # this command may require root access
```
LZ4's `Makefile` supports standard [Makefile conventions],
including [staged installs], [redirection], or [command redefinition].
It is compatible with parallel builds (`-j#`).
[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html

14
LICENSE
View File

@ -3,13 +3,9 @@ This repository uses 2 different licenses :
- all other files use a GPLv2 license, unless explicitly stated otherwise
Relevant license is reminded at the top of each source file,
and with the presence of COPYING or LICENSE file.
and with presence of COPYING or LICENSE file in associated directories.
This model emphasizes the fact that
only files in the `lib` directory are designed to be included into 3rd party projects.
Other files, such as those from `programs` or `examples` directory,
are not intended to be compiled outside of their context.
They can serve as source of inspiration,
but they should not be copy/pasted into 3rd party projects,
as this scenario is not supported.
This model is selected to emphasize that
files in the `lib` directory are designed to be included into 3rd party applications,
while all other files, in `programs`, `tests` or `examples`,
receive more limited attention and support for such scenario.

131
Makefile
View File

@ -3,6 +3,8 @@
# Copyright (C) Yann Collet 2011-2016
# All rights reserved.
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
# BSD license
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
@ -26,16 +28,10 @@
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# You can contact the author at :
# - LZ4 source repository : https://github.com/Cyan4973/lz4
# - LZ4 source repository : https://github.com/lz4/lz4
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
DESTDIR ?=
PREFIX ?= /usr/local
VOID := /dev/null
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
LZ4DIR = lib
PRGDIR = programs
TESTDIR = tests
@ -44,62 +40,91 @@ EXDIR = examples
# Define nul output
ifneq (,$(filter Windows%,$(OS)))
EXT = .exe
EXT = .exe
VOID = nul
else
EXT =
EXT =
VOID = /dev/null
endif
.PHONY: default all lib lz4 clean test versionsTest examples
.PHONY: default
default: lib-release lz4-release
default: lib lz4-release
.PHONY: all
all: allmost manuals
all:
.PHONY: allmost
allmost: lib lz4 examples
.PHONY: lib lib-release liblz4.a
lib lib-release liblz4.a:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
@$(MAKE) -C $(TESTDIR) $@
@$(MAKE) -C $(EXDIR) $@
lib:
@$(MAKE) -C $(LZ4DIR)
lz4:
.PHONY: lz4 lz4-release
lz4 : liblz4.a
lz4-release : lib-release
lz4 lz4-release :
@$(MAKE) -C $(PRGDIR) $@
@cp $(PRGDIR)/lz4$(EXT) .
lz4-release:
@$(MAKE) -C $(PRGDIR)
@cp $(PRGDIR)/lz4$(EXT) .
.PHONY: examples
examples: lib lz4
$(MAKE) -C $(EXDIR) test
.PHONY: manuals
manuals:
@$(MAKE) -C contrib/gen_manual $@
.PHONY: clean
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(EXDIR) $@ > $(VOID)
@$(MAKE) -C examples $@ > $(VOID)
@$(MAKE) -C contrib/gen_manual $@ > $(VOID)
@$(RM) lz4$(EXT)
@echo Cleaning completed
#------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX
install:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
uninstall:
.PHONY: install uninstall
install uninstall:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
travis-install:
$(MAKE) install PREFIX=~/install_test_dir
$(MAKE) -j1 install DESTDIR=~/install_test_dir
cmake:
@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
endif
#------------------------------------------------------------------------
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) test
$(MAKE) -C $(TESTDIR) $@
clangtest: clean
clang -v
@ -130,31 +155,10 @@ platformTest: clean
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all
$(MAKE) -C $(TESTDIR) test-platform
.PHONY: versionsTest
versionsTest: clean
$(MAKE) -C $(TESTDIR) $@
examples:
$(MAKE) -C $(LZ4DIR)
$(MAKE) -C $(PRGDIR) lz4
$(MAKE) -C examples test
endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
endif
#------------------------------------------------------------------------
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
cmake:
@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
gpptest: clean
g++ -v
CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
@ -168,13 +172,10 @@ gpptest32: clean
CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
c_standards: clean
$(MAKE) all MOREFLAGS="-std=gnu90 -Werror"
$(MAKE) clean
$(MAKE) all MOREFLAGS="-std=c99 -Werror"
$(MAKE) clean
$(MAKE) all MOREFLAGS="-std=gnu99 -Werror"
$(MAKE) clean
$(MAKE) all MOREFLAGS="-std=c11 -Werror"
$(MAKE) clean
# note : lz4 is not C90 compatible, because it requires long long support
CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost
CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost
CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost
endif

42
NEWS
View File

@ -1,7 +1,47 @@
v1.8.1
perf : faster and stronger ultra modes (levels 10+)
perf : slightly faster compression and decompression speed
perf : fix bad degenerative case, reported by @c-morgenstern
fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid)
cli : support for dictionary compression (`-D`), by Felix Handte @felixhandte
cli : fix : `lz4 -d --rm` preserves timestamp (#441)
cli : fix : do not modify /dev/null permission as root, by @aliceatlas
api : `_destSize()` variant supported for all compression levels
build : `make` and `make test` compatible with `-jX`, reported by @mwgamera
build : can control LZ4LIB_VISIBILITY macro, by @mikir
install: fix man page directory (#387), reported by Stuart Cardall (@itoffshore)
v1.8.0
cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
cli : added GNU separator -- specifying that all following arguments are files
API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
API : added LZ4F_resetDecompressionContext()
API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
API : lz4frame : can control block checksum and dictionary ID
API : fix : expose obsolete decoding functions, reported by Chen Yufei
API : experimental : lz4frame_static : new dictionary compression API
build : fix : static lib installation, by Ido Rosen
build : dragonFlyBSD, OpenBSD, NetBSD supported
build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
doc : Updated LZ4 Frame format to v1.6.0, restoring Dictionary-ID field
doc : lz4 api manual, by Przemyslaw Skibinski
v1.7.5
lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski
lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z)
cli : fix minor notification when using -r recursive mode
API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280)
doc : markdown version of man page, by Takayuki Matsuoka (#279)
build : Makefile : fix make -jX lib+exe concurrency (#277)
build : cmake : improvements by Michał Górny (#296)
v1.7.4.2
fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin)
v1.7.4
Improved : much better speed in -mx32 mode
cli : fix : Large file support in 32-bits mode on Mac OS-X
fix : compilation on gcc 4.4 (#282), reported by Antoine Martin
fix : compilation on gcc 4.4 (#272), reported by Antoine Martin
v1.7.3
Changed : moved to versioning; package, cli and library have same version number

25
README.md
View File

@ -25,14 +25,13 @@ LZ4 library is provided as open-source software using BSD 2-Clause license.
[travisMasterBadge]: https://travis-ci.org/lz4/lz4.svg?branch=master "Continuous Integration test suite"
[travisDevBadge]: https://travis-ci.org/lz4/lz4.svg?branch=dev "Continuous Integration test suite"
[travisLink]: https://travis-ci.org/lz4/lz4
[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/jc2yhgwyc7qqtsko/branch/master?svg=true "Windows test suite"
[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/jc2yhgwyc7qqtsko/branch/dev?svg=true "Windows test suite"
[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=master&svg=true "Windows test suite"
[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=dev&svg=true "Windows test suite"
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/lz4-1lndh
[coverBadge]: https://scan.coverity.com/projects/4735/badge.svg "Static code analysis of Master branch"
[coverlink]: https://scan.coverity.com/projects/4735
> **Branch Policy:**
> - The "master" branch is considered stable, at all times.
> - The "dev" branch is the one where all contributions must be merged
before being promoted to master.
@ -70,7 +69,25 @@ in single-thread mode.
[zlib]: http://www.zlib.net/
[Zstandard]: http://www.zstd.net/
LZ4 is also compatible and well optimized for x32 mode, for which it provides +10% speed performance.
LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance.
Installation
-------------------------
```
make
make install # this command may require root access
```
LZ4's `Makefile` supports standard [Makefile conventions],
including [staged installs], [redirection], or [command redefinition].
It is compatible with parallel builds (`-j#`).
[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html
Documentation

2
appveyor.yml
View File

@ -82,7 +82,7 @@ build_script:
ECHO *** &&
ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&

38
circle.yml Normal file
View File

@ -0,0 +1,38 @@
dependencies:
override:
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
- sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind
test:
override:
# Tests compilers and C standards
- clang -v; make clangtest && make clean
- g++ -v; make gpptest && make clean
- gcc -v; make c_standards && make clean
- gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean
- gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean
- gcc-6 -v; make c_standards CC=gcc-6 && make clean
- gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
# Shorter tests
- make cmake && make clean
- make -C tests test-lz4
- make -C tests test-lz4c
- make -C tests test-frametest
- make -C tests test-fullbench
- make -C tests test-fuzzer && make clean
- make -C lib all && make clean
- pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
- make travis-install && make clean
# Longer tests
- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
- make usan && make clean
- clang -v; make staticAnalyze && make clean
# Valgrind tests
- make -C tests test-mem && make clean
# ARM, AArch64, PowerPC, PowerPC64 tests
- make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean
- make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean
- make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean
- make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static && make clean

2
contrib/cmake_unofficial/.gitignore vendored
View File

@ -5,3 +5,5 @@ CMakeFiles
*.cmake
Makefile
liblz4.pc
lz4c
install_manifest.txt

70
contrib/cmake_unofficial/CMakeLists.txt
View File

@ -65,6 +65,11 @@ endif(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED)
# which case we always use static libraries.
include(CMakeDependentOption)
CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT LZ4_BUNDLED_MODE" OFF)
CMAKE_DEPENDENT_OPTION(BUILD_STATIC_LIBS "Build static libraries" OFF "BUILD_SHARED_LIBS" ON)
if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS)
message(FATAL_ERROR "Both BUILD_SHARED_LIBS and BUILD_STATIC_LIBS have been disabled")
endif()
set(LZ4_LIB_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/lib")
set(LZ4_PROG_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/programs")
@ -90,28 +95,41 @@ set(LZ4_CLI_SOURCES
# we're building a shared library this is ignored and PIC is always
# used.
option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON)
if(LZ4_POSITION_INDEPENDENT_LIB OR BUILD_SHARED_LIBS)
set(LZ4_POSITION_INDEPENDENT_CODE TRUE)
else()
set(LZ4_POSITION_INDEPENDENT_CODE FALSE)
endif()
# liblz4
add_library(lz4 ${LZ4_SOURCES})
set_target_properties(lz4 PROPERTIES
SOVERSION "${LZ4_VERSION_STRING}"
VERSION "${LZ4_VERSION_STRING}"
POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE})
set(LZ4_LIBRARIES_BUILT)
if(BUILD_SHARED_LIBS)
add_library(lz4_shared SHARED ${LZ4_SOURCES})
set_target_properties(lz4_shared PROPERTIES
OUTPUT_NAME lz4
SOVERSION "${LZ4_VERSION_MAJOR}"
VERSION "${LZ4_VERSION_STRING}")
list(APPEND LZ4_LIBRARIES_BUILT lz4_shared)
endif()
if(BUILD_STATIC_LIBS)
add_library(lz4_static STATIC ${LZ4_SOURCES})
set_target_properties(lz4_static PROPERTIES
OUTPUT_NAME lz4
POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB})
list(APPEND LZ4_LIBRARIES_BUILT lz4_static)
endif()
# link to shared whenever possible, to static otherwise
if(BUILD_SHARED_LIBS)
set(LZ4_LINK_LIBRARY lz4_shared)
else()
set(LZ4_LINK_LIBRARY lz4_static)
endif()
# lz4
add_executable(lz4cli ${LZ4_CLI_SOURCES})
set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4)
target_link_libraries(lz4cli lz4)
target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY})
# lz4c
add_executable(lz4c ${LZ4_CLI_SOURCES})
set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS")
target_link_libraries(lz4c lz4)
target_link_libraries(lz4c ${LZ4_LINK_LIBRARY})
# Extra warning flags
include (CheckCCompilerFlag)
@ -149,16 +167,38 @@ if(NOT LZ4_BUNDLED_MODE)
install(TARGETS lz4cli lz4c
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(TARGETS lz4
install(TARGETS ${LZ4_LIBRARIES_BUILT}
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(FILES
"${LZ4_LIB_SOURCE_DIR}/lz4.h"
"${LZ4_LIB_SOURCE_DIR}/lz4frame.h"
"${LZ4_LIB_SOURCE_DIR}/lz4hc.h"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
install(FILES "${LZ4_PROG_SOURCE_DIR}/lz4.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
# install lz4cat and unlz4 symlinks on *nix
if(UNIX)
install(CODE "
foreach(f lz4cat unlz4)
set(dest \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_BINDIR}/\${f}\")
message(STATUS \"Symlinking: \${dest} -> lz4\")
execute_process(
COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\")
endforeach()
")
# create manpage aliases
foreach(f lz4cat unlz4)
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" ".so man1/lz4.1\n")
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${f}.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
endforeach()
endif(UNIX)
endif(NOT LZ4_BUNDLED_MODE)
# pkg-config
@ -176,4 +216,6 @@ else()
set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
endif()
# for liblz4.pc substitution
set(VERSION ${LZ4_VERSION_STRING})
configure_file(${LZ4_LIB_SOURCE_DIR}/liblz4.pc.in liblz4.pc @ONLY)

2
contrib/gen_manual/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# build artefact
gen_manual

26
contrib/gen_manual/Makefile
View File

@ -35,7 +35,15 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -W
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
LZ4API = ../../lib/lz4.h
LZ4MANUAL = ../../doc/lz4_manual.html
LZ4FAPI = ../../lib/lz4frame.h
LZ4FMANUAL = ../../doc/lz4frame_manual.html
LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LZ4VER := $(shell echo $(LIBVER_SCRIPT))
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@ -45,14 +53,24 @@ EXT =
endif
.PHONY: default gen_manual
.PHONY: default
default: gen_manual
gen_manual: gen_manual.cpp
$(CXX) $(FLAGS) $^ -o $@$(EXT)
$(CXX) $(FLAGS) $^ -o $@$(EXT)
$(LZ4MANUAL) : gen_manual $(LZ4API)
echo "Update lz4 manual in /doc"
./gen_manual $(LZ4VER) $(LZ4API) $@
$(LZ4FMANUAL) : gen_manual $(LZ4FAPI)
echo "Update lz4frame manual in /doc"
./gen_manual $(LZ4VER) $(LZ4FAPI) $@
.PHONY: manuals
manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL)
.PHONY: clean
clean:
@$(RM) gen_manual$(EXT)
@echo Cleaning completed

3
contrib/gen_manual/gen-lz4-manual.sh
View File

@ -6,4 +6,5 @@ LIBVER_PATCH_SCRIPT=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][
LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT
echo LZ4_VERSION=$LIBVER_SCRIPT
./gen_manual $LIBVER_SCRIPT ../../lib/lz4.h ./lz4_manual.html
./gen_manual "lz4 $LIBVER_SCRIPT" ../../lib/lz4.h ./lz4_manual.html
./gen_manual "lz4frame $LIBVER_SCRIPT" ../../lib/lz4frame.h ./lz4frame_manual.html

58
contrib/gen_manual/gen_manual.cpp
View File

@ -89,11 +89,13 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator)
/* print line with LZ4LIB_API removed and C++ comments not bold */
void print_line(stringstream &sout, string line)
{
size_t spos;
size_t spos, epos;
if (line.substr(0,11) == "LZ4LIB_API ") line = line.substr(11);
if (line.substr(0,12) == "LZ4FLIB_API ") line = line.substr(12);
spos = line.find("/*");
if (spos!=string::npos) {
epos = line.find("*/");
if (spos!=string::npos && epos!=string::npos) {
sout << line.substr(0, spos);
sout << "</b>" << line.substr(spos) << "<b>" << endl;
} else {
@ -118,7 +120,7 @@ int main(int argc, char *argv[]) {
return 1;
}
version = "lz4 " + string(argv[1]) + " Manual";
version = string(argv[1]) + " Manual";
istream.open(argv[2], ifstream::in);
if (!istream.is_open()) {
@ -158,36 +160,28 @@ int main(int argc, char *argv[]) {
continue;
}
/* comments of type /*= and /**= mean: use a <H3> header and show also all functions until first empty line */
if ((line.substr(0,3) == "/*=" || line.substr(0,4) == "/**=") && line.find("*/")!=string::npos) {
trim_comments(line);
trim(line, "= ");
sout << "<h3>" << line << "</h3><pre><b>";
lines = get_lines(input, ++linenum, "");
for (l=0; l<lines.size(); l++) {
print_line(sout, lines[l]);
}
sout << "</b></pre><BR>" << endl;
continue;
spos = line.find("/**=");
if (spos==string::npos) {
spos = line.find("/*!");
if (spos==string::npos)
spos = line.find("/**");
if (spos==string::npos)
spos = line.find("/*-");
if (spos==string::npos)
spos = line.find("/*=");
if (spos==string::npos)
continue;
exclam = line[spos+2];
}
else exclam = '=';
spos = line.find("/*!");
if (spos==string::npos)
spos = line.find("/**");
if (spos==string::npos)
spos = line.find("/*-");
if (spos==string::npos)
continue;
exclam = line[spos+2];
comments = get_lines(input, linenum, "*/");
if (!comments.empty()) comments[0] = line.substr(spos+3);
if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/"));
for (l=0; l<comments.size(); l++) {
if (comments[l].find(" *")==0) comments[l] = comments[l].substr(2);
else if (comments[l].find(" *")==0) comments[l] = comments[l].substr(3);
trim(comments[l], "*-");
trim(comments[l], "*-=");
}
while (!comments.empty() && comments[comments.size()-1].empty()) comments.pop_back(); // remove empty line at the end
while (!comments.empty() && comments[0].empty()) comments.erase(comments.begin()); // remove empty line at the start
@ -208,6 +202,18 @@ int main(int argc, char *argv[]) {
print_line(sout, comments[l]);
}
sout << "</p></pre><BR>" << endl << endl;
} else if (exclam == '=') { /* comments of type /*= and /**= mean: use a <H3> header and show also all functions until first empty line */
trim(comments[0], " ");
sout << "<h3>" << comments[0] << "</h3><pre>";
for (l=1; l<comments.size(); l++) {
print_line(sout, comments[l]);
}
sout << "</pre><b><pre>";
lines = get_lines(input, ++linenum, "");
for (l=0; l<lines.size(); l++) {
print_line(sout, lines[l]);
}
sout << "</pre></b><BR>" << endl;
} else { /* comments of type /** and /*- mean: this is a comment; use a <H2> header for the first line */
if (comments.empty()) continue;
@ -238,4 +244,4 @@ int main(int argc, char *argv[]) {
ostream << "</html>" << endl << "</body>" << endl;
return 0;
}
}

16
doc/lz4_Block_format.md
View File

@ -90,10 +90,18 @@ A 255 value means there is another byte to read and add.
There is no limit to the number of optional bytes that can be output this way.
(This points towards a maximum achievable compression ratio of about 250).
With the offset and the matchlength,
the decoder can now proceed to copy the data from the already decoded buffer.
On decoding the matchlength, we reach the end of the compressed sequence,
and therefore start another one.
Decoding the matchlength reaches the end of current sequence.
Next byte will be the start of another sequence.
But before moving to next sequence,
it's time to use the decoded match position and length.
The decoder copies matchlength bytes from match position to current position.
In some cases, matchlength is larger than offset.
Therefore, match pos + match length > current pos,
which means that later bytes to copy are not yet decoded.
This is called an "overlap match", and must be handled with special care.
The most common case is an offset of 1,
meaning the last byte is repeated matchlength times.
Parsing restrictions

84
doc/lz4_Frame_format.md
View File

@ -1,7 +1,7 @@
LZ4 Frame Format Description
============================
###Notices
### Notices
Copyright (c) 2013-2015 Yann Collet
@ -14,9 +14,9 @@ and that any substantive changes or deletions from the original
are clearly marked.
Distribution of this document is unlimited.
###Version
### Version
1.5.1 (31/03/2015)
1.6.0 (08/08/2017)
Introduction
@ -63,7 +63,7 @@ General Structure of LZ4 Frame format
| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum |
|:-------:|:-------------:| ----- | ----- | ------- | ----------- |
| 4 bytes | 3-11 bytes | | | 4 bytes | 0-4 bytes |
| 4 bytes | 3-15 bytes | | | 4 bytes | 0-4 bytes |
__Magic Number__
@ -72,7 +72,7 @@ Value : 0x184D2204
__Frame Descriptor__
3 to 11 Bytes, to be detailed in the next part.
3 to 15 Bytes, to be detailed in the next part.
Most important part of the spec.
__Data Blocks__
@ -118,31 +118,31 @@ to decode all concatenated frames in their sequential order.
Frame Descriptor
----------------
| FLG | BD | (Content Size) | HC |
| ------- | ------- |:--------------:| ------- |
| 1 byte | 1 byte | 0 - 8 bytes | 1 byte |
| FLG | BD | (Content Size) | (Dictionary ID) | HC |
| ------- | ------- |:--------------:|:---------------:| ------- |
| 1 byte | 1 byte | 0 - 8 bytes | 0 - 4 bytes | 1 byte |
The descriptor uses a minimum of 3 bytes,
and up to 11 bytes depending on optional parameters.
and up to 15 bytes depending on optional parameters.
__FLG byte__
| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 |
| ------- | ------- | ------- | --------- | ------- | --------- | -------- |
|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*|
| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1 | 0 |
| ------- |-------|-------|----------|------|----------|----------|------|
|FieldName|Version|B.Indep|B.Checksum|C.Size|C.Checksum|*Reserved*|DictID|
__BD byte__
| BitNb | 7 | 6-5-4 | 3-2-1-0 |
| ------- | -------- | ------------ | -------- |
|FieldName|*Reserved*| Block MaxSize|*Reserved*|
| BitNb | 7 | 6-5-4 | 3-2-1-0 |
| ------- | -------- | ------------- | -------- |
|FieldName|*Reserved*| Block MaxSize |*Reserved*|
In the tables, bit 7 is highest bit, while bit 0 is lowest.
__Version Number__
2-bits field, must be set to “01”.
2-bits field, must be set to `01`.
Any other value cannot be decoded by this version of the specification.
Other version numbers will use different flag layouts.
@ -154,7 +154,7 @@ If this flag is set to “0”, each block depends on previous ones
In such case, it’s necessary to decode all blocks in sequence.
Block dependency improves compression ratio, especially for small blocks.
On the other hand, it makes direct jumps or multi-threaded decoding impossible.
On the other hand, it makes random access or multi-threaded decoding impossible.
__Block checksum flag__
@ -172,13 +172,17 @@ Content Size usage is optional.
__Content checksum flag__
If this flag is set, a content checksum will be appended after the EndMark.
If this flag is set, a 32-bits content checksum will be appended
after the EndMark.
Recommended value : “1” (content checksum is present)
__Dictionary ID flag__
If this flag is set, a 4-bytes Dict-ID field will be present,
after the descriptor flags and the Content Size.
__Block Maximum Size__
This information is intended to help the decoder allocate memory.
This information is useful to help the decoder allocate memory.
Size here refers to the original (uncompressed) data size.
Block Maximum Size is one value among the following table :
@ -186,17 +190,17 @@ Block Maximum Size is one value among the following table :
| --- | --- | --- | --- | ----- | ------ | ---- | ---- |
| N/A | N/A | N/A | N/A | 64 KB | 256 KB | 1 MB | 4 MB |
The decoder may refuse to allocate block sizes above a (system-specific) size.
The decoder may refuse to allocate block sizes above any system-specific size.
Unused values may be used in a future revision of the spec.
A decoder conformant to the current version of the spec
is only able to decode blocksizes defined in this spec.
A decoder conformant with the current version of the spec
is only able to decode block sizes defined in this spec.
__Reserved bits__
Value of reserved bits **must** be 0 (zero).
Reserved bit might be used in a future version of the specification,
typically enabling new optional features.
If this happens, a decoder respecting the current version of the specification
When this happens, a decoder respecting the current specification version
shall not be able to decode such a frame.
__Content Size__
@ -208,12 +212,32 @@ Format is Little endian.
This value is informational, typically for display or memory allocation.
It can be skipped by a decoder, or used to validate content correctness.
__Dictionary ID__
Dict-ID is only present if the associated flag is set.
It's an unsigned 32-bits value, stored using little-endian convention.
A dictionary is useful to compress short input sequences.
The compressor can take advantage of the dictionary context
to encode the input in a more compact manner.
It works as a kind of “known prefix” which is used by
both the compressor and the decompressor to “warm-up” reference tables.
The decompressor can use Dict-ID identifier to determine
which dictionary must be used to correctly decode data.
The compressor and the decompressor must use exactly the same dictionary.
It's presumed that the 32-bits dictID uniquely identifies a dictionary.
Within a single frame, a single dictionary can be defined.
When the frame descriptor defines independent blocks,
each block will be initialized with the same dictionary.
If the frame descriptor defines linked blocks,
the dictionary will only be used once, at the beginning of the frame.
__Header Checksum__
One-byte checksum of combined descriptor fields, including optional ones.
The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF `
using zero as a seed,
and the full Frame Descriptor as an input
The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF `
using zero as a seed, and the full Frame Descriptor as an input
(including optional fields when they are present).
A wrong checksum indicates an error in the descriptor.
Header checksum is informational and can be skipped.
@ -347,7 +371,7 @@ wether it is a file or a stream.
Alternatively, if the frame is followed by a valid Frame Magic Number,
it is considered completed.
It makes legacy frames compatible with frame concatenation.
This policy makes it possible to concatenate legacy frames.
Any other value will be interpreted as a block size,
and trigger an error if it does not fit within acceptable range.
@ -356,7 +380,9 @@ and trigger an error if it does not fit within acceptable range.
Version changes
---------------
1.5.1 : changed format to MarkDown compatible
1.6.0 : restored Dictionary ID field in Frame header
1.5.1 : changed document format to MarkDown
1.5 : removed Dictionary ID from specification

447
doc/lz4_manual.html
View File

@ -1,20 +1,22 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>lz4 1.7.2 Manual</title>
<title>1.8.1 Manual</title>
</head>
<body>
<h1>lz4 1.7.2 Manual</h1>
<h1>1.8.1 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
<li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Tuning parameter</a></li>
<li><a href="#Chapter3">Private definitions</a></li>
<li><a href="#Chapter2">Version</a></li>
<li><a href="#Chapter3">Tuning parameter</a></li>
<li><a href="#Chapter4">Simple Functions</a></li>
<li><a href="#Chapter5">Advanced Functions</a></li>
<li><a href="#Chapter6">Streaming Compression Functions</a></li>
<li><a href="#Chapter7">Streaming Decompression Functions</a></li>
<li><a href="#Chapter8">Private definitions</a></li>
<li><a href="#Chapter9">Obsolete Functions</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@ -29,19 +31,26 @@
- unbounded multiple steps (described as Streaming compression)
lz4.h provides block compression functions. It gives full buffer control to user.
Block compression functions are not-enough to send information,
since it's still necessary to provide metadata (such as compressed size),
and each application can do it in whichever way it wants.
For interoperability, there is LZ4 frame specification (doc/lz4_Frame_format.md).
Decompressing an lz4-compressed block also requires metadata (such as compressed size).
Each application is free to encode such metadata in whichever way it wants.
An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md),
take care of encoding standard metadata alongside LZ4-compressed blocks.
If your application requires interoperability, it's recommended to use it.
A library is provided to take care of it, see lz4frame.h.
<BR></pre>
<h3>Version</h3><pre><b>int LZ4_versionNumber (void);
const char* LZ4_versionString (void);
</b></pre><BR>
<a name="Chapter2"></a><h2>Tuning parameter</h2><pre></pre>
<a name="Chapter2"></a><h2>Version</h2><pre></pre>
<pre><b>#define LZ4_MEMORY_USAGE 14
<pre><b>int LZ4_versionNumber (void); </b>/**< library version number; to be used when checking dll version */<b>
</b></pre><BR>
<pre><b>const char* LZ4_versionString (void); </b>/**< library version string; to be used when checking dll version */<b>
</b></pre><BR>
<a name="Chapter3"></a><h2>Tuning parameter</h2><pre></pre>
<pre><b>#ifndef LZ4_MEMORY_USAGE
# define LZ4_MEMORY_USAGE 14
#endif
</b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
Increasing memory usage improves compression ratio
Reduced memory usage can improve speed, due to cache effect
@ -49,10 +58,191 @@ const char* LZ4_versionString (void);
</p></pre><BR>
<a name="Chapter3"></a><h2>Private definitions</h2><pre>
<a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre>
<pre><b>int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
</b><p> Compresses 'srcSize' bytes from buffer 'src'
into already allocated 'dst' buffer of size 'dstCapacity'.
Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'src' into a limited 'dst' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dst' content is not valid.
This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
srcSize : supported max value is LZ4_MAX_INPUT_VALUE
dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
or 0 if compression fails
</p></pre><BR>
<pre><b>int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
</b><p> compressedSize : is the exact complete size of the compressed block.
dstCapacity : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
If destination buffer is not large enough, decoding will stop and output an error code (negative value).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
</p></pre><BR>
<a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre>
<pre><b>int LZ4_compressBound(int inputSize);
</b><p> Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
</p></pre><BR>
<pre><b>int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
</b><p> Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
</p></pre><BR>
<pre><b>int LZ4_sizeofState(void);
int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
</b><p> Same compression function, just using an externally allocated memory space to store compression state.
Use LZ4_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
</p></pre><BR>
<pre><b>int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
</b><p> Reverse the logic : compresses as much data as possible from 'src' buffer
into already allocated buffer 'dst' of size 'targetDestSize'.
This function either compresses the entire 'src' content into 'dst' if it's large enough,
or fill 'dst' buffer completely with as much data as possible from 'src'.
*srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
New value is necessarily <= old value.
return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
or 0 if compression fails
</p></pre><BR>
<pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
</b><p> originalSize : is the original uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size)
If the source stream is detected malformed, the function will stop decoding and return a negative result.
Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
note : This function respects memory boundaries for *properly formed* compressed data.
It is a bit faster than LZ4_decompress_safe().
However, it does not provide any protection against intentionally modified data stream (malicious input).
Use this function in trusted environment only (data to decode comes from a trusted source).
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
</b><p> This function decompress a compressed block of size 'srcSize' at position 'src'
into destination buffer 'dst' of size 'dstCapacity'.
The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
@return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
</p></pre><BR>
<a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre>
<pre><b>LZ4_stream_t* LZ4_createStream(void);
int LZ4_freeStream (LZ4_stream_t* streamPtr);
</b><p> LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
LZ4_freeStream() releases its memory.
</p></pre><BR>
<pre><b>void LZ4_resetStream (LZ4_stream_t* streamPtr);
</b><p> An LZ4_stream_t structure can be allocated once and re-used multiple times.
Use this function to start compressing a new stream.
</p></pre><BR>
<pre><b>int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
</b><p> Use this function to load a static dictionary into LZ4_stream_t.
Any previous data will be forgotten, only 'dictionary' will remain in memory.
Loading a size of 0 is allowed, and is the same as reset.
@return : dictionary size, in bytes (necessarily <= 64 KB)
</p></pre><BR>
<pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
</b><p> Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
'dst' buffer must be already allocated.
If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
@return : size of compressed block
or 0 if there is an error (typically, compressed data cannot fit into 'dst')
After an error, the stream status is invalid, it can only be reset or freed.
</p></pre><BR>
<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
</b><p> If previously compressed data block is not guaranteed to remain available at its current memory location,
save it into a safer place (char* safeBuffer).
Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
@return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
</p></pre><BR>
<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre> Bufferless synchronous API
<BR></pre>
<pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void);
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
</b><p> creation / destruction of streaming decompression tracking structure.
A tracking structure can be re-used multiple times sequentially.
</p></pre><BR>
<pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
</b><p> An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
Use this function to start decompression of a new stream of blocks.
A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
@return : 1 if OK, 0 if error
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
</b><p> These decoding functions allow decompression of consecutive blocks in "streaming" mode.
A block is an unsplittable entity, it must be presented entirely to a decompression function.
Decompression functions only accept one block at a time.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size of any single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
</b><p> These decoding functions work the same as
a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
They are stand-alone, and don't need an LZ4_streamDecode_t structure.
</p></pre><BR>
<a name="Chapter8"></a><h2>Private definitions</h2><pre>
Do not use these definitions.
They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
If you use these definitions in your code, it will break when you upgrade LZ4 to a new version.
Using these definitions will expose code to API and/or ABI break in future versions of the library.
<BR></pre>
<pre><b>typedef struct {
@ -87,201 +277,60 @@ const char* LZ4_versionString (void);
size_t prefixSize;
} LZ4_streamDecode_t_internal;
</b></pre><BR>
<a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre>
<pre><b>int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
</b><p> Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails
</p></pre><BR>
<pre><b>int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
</b><p> compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
</p></pre><BR>
<a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre>
<pre><b>int LZ4_compressBound(int inputSize);
</b><p> Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
</p></pre><BR>
<pre><b>int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
</b><p> Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
</p></pre><BR>
<pre><b>int LZ4_sizeofState(void);
int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
</b><p> Same compression function, just using an externally allocated memory space to store compression state.
Use LZ4_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
</p></pre><BR>
<pre><b>int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
</b><p> Reverse the logic, by compressing as much data as possible from 'source' buffer
into already allocated buffer 'dest' of size 'targetDestSize'.
This function either compresses the entire 'source' content into 'dest' if it's large enough,
or fill 'dest' buffer completely with as much data as possible from 'source'.
*sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
New value is necessarily <= old value.
return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
or 0 if compression fails
</p></pre><BR>
<pre><b>int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
</b><p> originalSize : is the original and therefore uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size)
If the source stream is detected malformed, the function will stop decoding and return a negative result.
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
note : This function fully respect memory boundaries for properly formed compressed data.
It is a bit faster than LZ4_decompress_safe().
However, it does not provide any protection against intentionally modified data stream (malicious input).
Use this function in trusted environment only (data to decode comes from a trusted source).
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
</b><p> This function decompress a compressed block of size 'compressedSize' at position 'source'
into destination buffer 'dest' of size 'maxDecompressedSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
</p></pre><BR>
<a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre>
<pre><b>typedef struct {
union {
long long table[LZ4_STREAMSIZE_U64];
<pre><b>#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
union LZ4_stream_u {
unsigned long long table[LZ4_STREAMSIZE_U64];
LZ4_stream_t_internal internal_donotuse;
};
} LZ4_stream_t;
} ; </b>/* previously typedef'd to LZ4_stream_t */<b>
</b><p> information structure to track an LZ4 stream.
important : init this structure content before first use !
note : only allocated directly the structure if you are statically linking LZ4
If you are using liblz4 as a DLL, please use below construction methods instead.
init this structure before first use.
note : only use in association with static linking !
this definition is not API/ABI safe,
it may change in a future version !
</p></pre><BR>
<pre><b>void LZ4_resetStream (LZ4_stream_t* streamPtr);
</b><p> Use this function to init an allocated `LZ4_stream_t` structure
</p></pre><BR>
<pre><b>LZ4_stream_t* LZ4_createStream(void);
int LZ4_freeStream (LZ4_stream_t* streamPtr);
</b><p> LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
LZ4_freeStream() releases its memory.
In the context of a DLL (liblz4), please use these methods rather than the static struct.
They are more future proof, in case of a change of `LZ4_stream_t` size.
</p></pre><BR>
<pre><b>int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
</b><p> Use this function to load a static dictionary into LZ4_stream.
Any previous data will be forgotten, only 'dictionary' will remain in memory.
Loading a size of 0 is allowed.
Return : dictionary size, in bytes (necessarily <= 64 KB)
</p></pre><BR>
<pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
</b><p> Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
Important : Previous data blocks are assumed to still be present and unmodified !
'dst' buffer must be already allocated.
If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
</p></pre><BR>
<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
</b><p> If previously compressed data block is not guaranteed to remain available at its memory location,
save it into a safer place (char* safeBuffer).
Note : you don't need to call LZ4_loadDict() afterwards,
dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
</p></pre><BR>
<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre></pre>
<pre><b>typedef struct {
union {
<pre><b>#define LZ4_STREAMDECODESIZE_U64 4
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
union LZ4_streamDecode_u {
unsigned long long table[LZ4_STREAMDECODESIZE_U64];
LZ4_streamDecode_t_internal internal_donotuse;
};
</b></pre><BR>
<pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void);
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
</b><p> information structure to track an LZ4 stream.
init this structure content using LZ4_setStreamDecode or memset() before first use !
In the context of a DLL (liblz4) please prefer usage of construction methods below.
They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
LZ4_freeStreamDecode releases its memory.
} ; </b>/* previously typedef'd to LZ4_streamDecode_t */<b>
</b><p> information structure to track an LZ4 stream during decompression.
init this structure using LZ4_setStreamDecode (or memset()) before first use
note : only use in association with static linking !
this definition is not API/ABI safe,
and may change in a future version !
</p></pre><BR>
<pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
</b><p> Use this function to instruct where to find the dictionary.
Setting a size of 0 is allowed (same effect as reset).
@return : 1 if OK, 0 if error
</p></pre><BR>
<a name="Chapter9"></a><h2>Obsolete Functions</h2><pre></pre>
<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ4_setStreamDecode()
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
</b><p>Advanced decoding functions :
These decoding functions work the same as
a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
<pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) </b>/* disable deprecation warnings */<b>
#else
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined(__clang__) </b>/* clang doesn't handle mixed C++11 and CNU attributes */<b>
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b>
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
# elif (LZ4_GCC_VERSION >= 405)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# endif
#endif </b>/* LZ4_DISABLE_DEPRECATE_WARNINGS */<b>
</b><p> Should deprecation warnings be a problem,
it is generally possible to disable them,
typically with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual.
Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS
</p></pre><BR>
</html>

282
doc/lz4frame_manual.html Normal file
View File

@ -0,0 +1,282 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>1.8.1 Manual</title>
</head>
<body>
<h1>1.8.1 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
<li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Compiler specifics</a></li>
<li><a href="#Chapter3">Error management</a></li>
<li><a href="#Chapter4">Frame compression types</a></li>
<li><a href="#Chapter5">Simple compression function</a></li>
<li><a href="#Chapter6">Advanced compression functions</a></li>
<li><a href="#Chapter7">Resource Management</a></li>
<li><a href="#Chapter8">Compression</a></li>
<li><a href="#Chapter9">Decompression functions</a></li>
<li><a href="#Chapter10">Streaming decompression functions</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
lz4frame.h provides frame compression functions that take care
of encoding standard metadata alongside LZ4-compressed blocks.
<BR></pre>
<a name="Chapter2"></a><h2>Compiler specifics</h2><pre></pre>
<a name="Chapter3"></a><h2>Error management</h2><pre></pre>
<pre><b>unsigned LZ4F_isError(LZ4F_errorCode_t code); </b>/**< tells if a `LZ4F_errorCode_t` function result is an error code */<b>
</b></pre><BR>
<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code); </b>/**< return error code string; useful for debugging */<b>
</b></pre><BR>
<a name="Chapter4"></a><h2>Frame compression types</h2><pre></pre>
<pre><b>typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
LZ4F_max256KB=5,
LZ4F_max1MB=6,
LZ4F_max4MB=7
LZ4F_OBSOLETE_ENUM(max64KB)
LZ4F_OBSOLETE_ENUM(max256KB)
LZ4F_OBSOLETE_ENUM(max1MB)
LZ4F_OBSOLETE_ENUM(max4MB)
} LZ4F_blockSizeID_t;
</b></pre><BR>
<pre><b>typedef enum {
LZ4F_blockLinked=0,
LZ4F_blockIndependent
LZ4F_OBSOLETE_ENUM(blockLinked)
LZ4F_OBSOLETE_ENUM(blockIndependent)
} LZ4F_blockMode_t;
</b></pre><BR>
<pre><b>typedef enum {
LZ4F_noContentChecksum=0,
LZ4F_contentChecksumEnabled
LZ4F_OBSOLETE_ENUM(noContentChecksum)
LZ4F_OBSOLETE_ENUM(contentChecksumEnabled)
} LZ4F_contentChecksum_t;
</b></pre><BR>
<pre><b>typedef enum {
LZ4F_noBlockChecksum=0,
LZ4F_blockChecksumEnabled
} LZ4F_blockChecksum_t;
</b></pre><BR>
<pre><b>typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
</b></pre><BR>
<pre><b>typedef struct {
LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b>
LZ4F_blockMode_t blockMode; </b>/* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */<b>
LZ4F_contentChecksum_t contentChecksumFlag; </b>/* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */<b>
LZ4F_frameType_t frameType; </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
unsigned long long contentSize; </b>/* Size of uncompressed content ; 0 == unknown */<b>
unsigned dictID; </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b>
LZ4F_blockChecksum_t blockChecksumFlag; </b>/* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */<b>
} LZ4F_frameInfo_t;
</b><p> makes it possible to set or read frame parameters.
It's not required to set all fields, as long as the structure was initially memset() to zero.
For all fields, 0 sets it to default value
</p></pre><BR>
<pre><b>typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b>
unsigned autoFlush; </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
unsigned reserved[4]; </b>/* must be zero for forward compatibility */<b>
} LZ4F_preferences_t;
</b><p> makes it possible to supply detailed compression parameters to the stream interface.
It's not required to set all fields, as long as the structure was initially memset() to zero.
All reserved fields must be set to zero.
</p></pre><BR>
<a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
<pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
</p></pre><BR>
<pre><b>size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
const LZ4F_preferences_t* preferencesPtr);
</b><p> Compress an entire srcBuffer into a valid LZ4 frame.
dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
@return : number of bytes written into dstBuffer.
or an error code if it fails (can be tested using LZ4F_isError())
</p></pre><BR>
<a name="Chapter6"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>typedef struct {
unsigned stableSrc; </b>/* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */<b>
unsigned reserved[3];
} LZ4F_compressOptions_t;
</b></pre><BR>
<a name="Chapter7"></a><h2>Resource Management</h2><pre></pre>
<pre><b>LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
</b><p> The first thing to do is to create a compressionContext object, which will be used in all compression operations.
This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
The function will provide a pointer to a fully allocated LZ4F_cctx object.
If @return != zero, there was an error during context creation.
Object can release its memory using LZ4F_freeCompressionContext();
</p></pre><BR>
<a name="Chapter8"></a><h2>Compression</h2><pre></pre>
<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
void* dstBuffer, size_t dstCapacity,
const LZ4F_preferences_t* prefsPtr);
</b><p> will write the frame header into dstBuffer.
dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
`prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
@return : number of bytes written into dstBuffer for the header
or an error code (which can be tested using LZ4F_isError())
</p></pre><BR>
<pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
</p></pre><BR>
<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
</b><p> LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
This value is provided by LZ4F_compressBound().
If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
`cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
@return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
or an error code if it fails (which can be tested using LZ4F_isError())
</p></pre><BR>
<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
</b><p> When data must be generated and sent immediately, without waiting for a block to be completely filled,
it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
`dstCapacity` must be large enough to ensure the operation will be successful.
`cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
@return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
or an error code if it fails (which can be tested using LZ4F_isError())
</p></pre><BR>
<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
</b><p> To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
It will flush whatever data remained within `cctx` (like LZ4_flush())
and properly finalize the frame, with an endMark and a checksum.
`cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
@return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
or an error code if it fails (which can be tested using LZ4F_isError())
A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
</p></pre><BR>
<a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre>
<pre><b>typedef struct {
unsigned stableDst; </b>/* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */<b>
unsigned reserved[3]; </b>/* must be set to zero for forward compatibility */<b>
} LZ4F_decompressOptions_t;
</b></pre><BR>
<pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
</b><p> Create an LZ4F_dctx object, to track all decompression operations.
The version provided MUST be LZ4F_VERSION.
The function provides a pointer to an allocated and initialized LZ4F_dctx object.
The result is an errorCode, which can be tested using LZ4F_isError().
dctx memory can be released using LZ4F_freeDecompressionContext();
The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
That is, it should be == 0 if decompression has been completed fully and correctly.
</p></pre><BR>
<a name="Chapter10"></a><h2>Streaming decompression functions</h2><pre></pre>
<pre><b>size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
</b><p> This function extracts frame parameters (max blockSize, dictID, etc.).
Its usage is optional.
Extracted information is typically useful for allocation and dictionary.
This function works in 2 situations :
- At the beginning of a new frame, in which case
it will decode information from `srcBuffer`, starting the decoding process.
Input size must be large enough to successfully decode the entire frame header.
Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
It's allowed to provide more input data than this minimum.
- After decoding has been started.
In which case, no input is read, frame parameters are extracted from dctx.
- If decoding has barely started, but not yet extracted information from header,
LZ4F_getFrameInfo() will fail.
The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
Decompression must resume from (srcBuffer + *srcSizePtr).
@return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
or an error code which can be tested using LZ4F_isError().
note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
</p></pre><BR>
<pre><b>size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* dOptPtr);
</b><p> Call this function repetitively to regenerate compressed data from `srcBuffer`.
The function will read up to *srcSizePtr bytes from srcBuffer,
and decompress data into dstBuffer, of capacity *dstSizePtr.
The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
Unconsumed source data must be presented again in subsequent invocations.
`dstBuffer` can freely change between each consecutive function invocation.
`dstBuffer` content will be overwritten.
@return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
Schematically, it's the size of the current (or remaining) compressed block + header of next block.
Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
This is just a hint though, it's always possible to provide any srcSize.
When a frame is fully decoded, @return will be 0 (no more data expected).
When provided with more bytes than necessary to decode a frame,
LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0.
If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
After a decompression error, the `dctx` context is not resumable.
Use LZ4F_resetDecompressionContext() to return to clean state.
After a frame is fully decoded, dctx can be used again to decompress another frame.
</p></pre><BR>
<pre><b>void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); </b>/* always successful */<b>
</b><p> In case of an error, the context is left in "undefined" state.
In which case, it's necessary to reset it, before re-using it.
This method can also be used to abruptly stop any unfinished decompression,
and start a new one using same context resources.
</p></pre><BR>
</html>
</body>

1
examples/.gitignore vendored
View File

@ -6,4 +6,5 @@
/ringBufferHC
/lineCompress
/frameCompress
/simpleBuffer
/*.exe

6
examples/HCStreaming_ringBuffer.c
View File

@ -1,12 +1,12 @@
// LZ4 HC streaming API example : ring buffer
// Based on previous work from Takayuki Matsuoka
// Based on a previous example by Takayuki Matsuoka
/**************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* for MSVC */
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif

27
examples/Makefile
View File

@ -27,13 +27,14 @@
# kindly provided by Takayuki Matsuoka
# ##########################################################################
CFLAGS ?= -O3
CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CPPFLAGS += -I../lib
CFLAGS ?= -O3
CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
FLAGS := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
TESTFILE= Makefile
LZ4DIR := ../lib
LZ4 = ../programs/lz4
TESTFILE = Makefile
LZ4DIR := ../lib
LZ4 = ../programs/lz4
# Define *.exe as extension for Windows systems
@ -48,7 +49,8 @@ endif
default: all
all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC lineCompress frameCompress
all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
lineCompress frameCompress simpleBuffer
printVersion: $(LZ4DIR)/lz4.c printVersion.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@ -78,12 +80,21 @@ simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
test : all
@echo "\n=== Print Version ==="
./printVersion$(EXT)
@echo "\n=== Simple compression example ==="
./simpleBuffer$(EXT)
@echo "\n=== Double-buffer ==="
./doubleBuffer$(EXT) $(TESTFILE)
./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400
@echo "\n=== Ring Buffer ==="
./ringBuffer$(EXT) $(TESTFILE)
@echo "\n=== Ring Buffer + LZ4 HC ==="
./ringBufferHC$(EXT) $(TESTFILE)
@echo "\n=== Compress line by line ==="
./lineCompress$(EXT) $(TESTFILE)
@echo "\n=== Dictionary Random Access ==="
./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400
@echo "\n=== Frame compression ==="
./frameCompress$(EXT) $(TESTFILE)
$(LZ4) -vt $(TESTFILE).lz4

2
examples/blockStreaming_doubleBuffer.c
View File

@ -2,7 +2,7 @@
// Copyright : Takayuki Matsuoka
#ifdef _MSC_VER /* Visual Studio */
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif

4
examples/blockStreaming_lineByLine.c
View File

@ -1,8 +1,8 @@
// LZ4 streaming API example : line-by-line logfile compression
// Copyright : Takayuki Matsuoka
// by Takayuki Matsuoka
#ifdef _MSC_VER /* Visual Studio */
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif

55
examples/blockStreaming_ringBuffer.c
View File

@ -1,17 +1,14 @@
// LZ4 streaming API example : ring buffer
// Based on sample code from Takayuki Matsuoka
/* LZ4 streaming API example : ring buffer
* Based on sample code from Takayuki Matsuoka */
/**************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
#endif
/**************************************
@ -27,7 +24,7 @@
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers
DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES /* Intentionally larger, to test unsynchronized ring buffers */
};
@ -50,7 +47,7 @@ size_t read_bin(FILE* fp, void* array, int arrayBytes) {
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_stream_t lz4Stream_body = { 0 };
LZ4_stream_t lz4Stream_body = { { 0 } };
LZ4_stream_t* lz4Stream = &lz4Stream_body;
static char inpBuf[RING_BUFFER_BYTES];
@ -85,24 +82,22 @@ void test_compress(FILE* outFp, FILE* inpFp)
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DECODE_RING_BUFFER];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { { 0 } };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
for(;;) {
int cmpBytes = 0;
char cmpBuf[CMPBUFSIZE];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
{ const size_t r0 = read_int32(inpFp, &cmpBytes);
if(r0 != 1 || cmpBytes <= 0) break;
const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes) break;
}
{
char* const decPtr = &decBuf[decOffset];
{ char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0) break;
@ -120,7 +115,7 @@ int compare(FILE* f0, FILE* f1)
{
int result = 0;
while(0 == result) {
while (0 == result) {
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
@ -128,12 +123,9 @@ int compare(FILE* f0, FILE* f1)
result = (int) r0 - (int) r1;
if(0 == r0 || 0 == r1) {
break;
}
if(0 == result) {
result = memcmp(b0, b1, r0);
}
if (0 == r0 || 0 == r1) break;
if (0 == result) result = memcmp(b0, b1, r0);
}
return result;
@ -146,7 +138,7 @@ int main(int argc, char** argv)
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
if(argc < 2) {
if (argc < 2) {
printf("Please specify input filename\n");
return 0;
}
@ -160,9 +152,8 @@ int main(int argc, char** argv)
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
test_compress(outFp, inpFp);
@ -171,9 +162,8 @@ int main(int argc, char** argv)
}
// decompress
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
@ -182,12 +172,11 @@ int main(int argc, char** argv)
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
if (0 == cmp) {
printf("Verify : OK\n");
} else {
printf("Verify : NG\n");

2
examples/dictionaryRandomAccess.c
View File

@ -1,6 +1,6 @@
// LZ4 API example : Dictionary Random Access
#ifdef _MSC_VER /* Visual Studio */
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif

488
examples/frameCompress.c
View File

@ -13,299 +13,303 @@
#define LZ4_FOOTER_SIZE 4
static const LZ4F_preferences_t lz4_preferences = {
{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } },
0, /* compression level */
0, /* autoflush */
{ 0, 0, 0, 0 }, /* reserved, must be set to 0 */
{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
0, /* compression level */
0, /* autoflush */
{ 0, 0, 0, 0 }, /* reserved, must be set to 0 */
};
static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
LZ4F_errorCode_t r;
LZ4F_compressionContext_t ctx;
char *src, *buf = NULL;
size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
size_t r=1; /* function result; 1 == error, default (early exit) */
LZ4F_compressionContext_t ctx;
char *src, *buf = NULL;
size_t size, count_in = 0, count_out, offset = 0, frame_size;
r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(r)) {
printf("Failed to create context: error %zu\n", r);
return 1;
}
r = 1;
if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
printf("Failed to create context: error %zu\n", r);
return 1;
}
src = malloc(BUF_SIZE);
if (!src) {
printf("Not enough memory\n");
goto cleanup;
}
src = malloc(BUF_SIZE);
if (!src) {
printf("Not enough memory\n");
goto cleanup;
}
frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
buf = malloc(size);
if (!buf) {
printf("Not enough memory\n");
goto cleanup;
}
frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
buf = malloc(size);
if (!buf) {
printf("Not enough memory\n");
goto cleanup;
}
n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
if (LZ4F_isError(n)) {
printf("Failed to start compression: error %zu\n", n);
goto cleanup;
}
{ size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
if (LZ4F_isError(headerSize)) {
printf("Failed to start compression: error %zu\n", headerSize);
goto cleanup;
}
offset = count_out = headerSize;
printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize);
}
printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
for (;;) {
k = fread(src, 1, BUF_SIZE, in);
if (k == 0)
break;
count_in += k;
for (;;) {
size_t const readSize = fread(src, 1, BUF_SIZE, in);
if (readSize == 0)
break;
count_in += readSize;
n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
if (LZ4F_isError(n)) {
printf("Compression failed: error %zu\n", n);
goto cleanup;
}
{ size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL);
if (LZ4F_isError(compressedSize)) {
printf("Compression failed: error %zu\n", compressedSize);
goto cleanup;
}
offset += compressedSize;
count_out += compressedSize;
}
offset += n;
count_out += n;
if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
printf("Writing %zu bytes\n", offset);
if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
size_t writtenSize;
printf("Writing %zu bytes\n", offset);
k = fwrite(buf, 1, offset, out);
if (k < offset) {
if (ferror(out))
printf("Write failed\n");
else
printf("Short write\n");
goto cleanup;
}
writtenSize = fwrite(buf, 1, offset, out);
if (writtenSize < offset) {
if (ferror(out)) /* note : ferror() must follow fwrite */
printf("Write failed\n");
else
printf("Short write\n");
goto cleanup;
}
offset = 0;
}
}
offset = 0;
}
}
n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
if (LZ4F_isError(n)) {
printf("Failed to end compression: error %zu\n", n);
goto cleanup;
}
{ size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
if (LZ4F_isError(compressedSize)) {
printf("Failed to end compression: error %zu\n", compressedSize);
goto cleanup;
}
offset += compressedSize;
count_out += compressedSize;
}
offset += n;
count_out += n;
printf("Writing %zu bytes\n", offset);
printf("Writing %zu bytes\n", offset);
{ size_t const writtenSize = fwrite(buf, 1, offset, out);
if (writtenSize < offset) {
if (ferror(out))
printf("Write failed\n");
else
printf("Short write\n");
goto cleanup;
} }
k = fwrite(buf, 1, offset, out);
if (k < offset) {
if (ferror(out))
printf("Write failed\n");
else
printf("Short write\n");
goto cleanup;
}
*size_in = count_in;
*size_out = count_out;
r = 0; /* success */
*size_in = count_in;
*size_out = count_out;
r = 0;
cleanup:
if (ctx)
LZ4F_freeCompressionContext(ctx);
free(src);
free(buf);
return r;
LZ4F_freeCompressionContext(ctx); /* supports free on NULL */
free(src);
free(buf);
return r;
}
static size_t get_block_size(const LZ4F_frameInfo_t* info) {
switch (info->blockSizeID) {
switch (info->blockSizeID) {
case LZ4F_default:
case LZ4F_max64KB: return 1 << 16;
case LZ4F_max256KB: return 1 << 18;
case LZ4F_max1MB: return 1 << 20;
case LZ4F_max4MB: return 1 << 22;
default:
printf("Impossible unless more block sizes are allowed\n");
exit(1);
}
case LZ4F_max64KB: return 1 << 16;
case LZ4F_max256KB: return 1 << 18;
case LZ4F_max1MB: return 1 << 20;
case LZ4F_max4MB: return 1 << 22;
default:
printf("Impossible unless more block sizes are allowed\n");
exit(1);
}
}
static size_t decompress_file(FILE *in, FILE *out) {
void* const src = malloc(BUF_SIZE);
void* dst = NULL;
size_t dstCapacity = 0;
LZ4F_dctx *dctx = NULL;
size_t ret;
static size_t decompress_file(FILE* in, FILE* out) {
void* const src = malloc(BUF_SIZE);
void* dst = NULL;
size_t dstCapacity = 0;
LZ4F_dctx* dctx = NULL;
size_t ret = 1;
/* Initialization */
/* Initialization */
if (!src) { perror("decompress_file(src)"); goto cleanup; }
ret = LZ4F_createDecompressionContext(&dctx, 100);
if (LZ4F_isError(ret)) {
printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
goto cleanup;
}
{ size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
if (LZ4F_isError(dctxStatus)) {
printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
goto cleanup;
} }
/* Decompression */
ret = 1;
while (ret != 0) {
/* Load more input */
size_t srcSize = fread(src, 1, BUF_SIZE, in);
void* srcPtr = src;
void* srcEnd = srcPtr + srcSize;
if (srcSize == 0 || ferror(in)) {
printf("Decompress: not enough input or error reading file\n");
goto cleanup;
}
/* Allocate destination buffer if it isn't already */
if (!dst) {
LZ4F_frameInfo_t info;
ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
if (LZ4F_isError(ret)) {
printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
goto cleanup;
}
/* Allocating enough space for an entire block isn't necessary for
* correctness, but it allows some memcpy's to be elided.
*/
dstCapacity = get_block_size(&info);
dst = malloc(dstCapacity);
/* Decompression */
while (ret != 0) {
/* Load more input */
size_t srcSize = fread(src, 1, BUF_SIZE, in);
const void* srcPtr = src;
const void* const srcEnd = srcPtr + srcSize;
if (srcSize == 0 || ferror(in)) {
printf("Decompress: not enough input or error reading file\n");
goto cleanup;
}
/* Allocate destination buffer if it isn't already */
if (!dst) {
LZ4F_frameInfo_t info;
ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
if (LZ4F_isError(ret)) {
printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
goto cleanup;
}
/* Allocating enough space for an entire block isn't necessary for
* correctness, but it allows some memcpy's to be elided.
*/
dstCapacity = get_block_size(&info);
dst = malloc(dstCapacity);
if (!dst) { perror("decompress_file(dst)"); goto cleanup; }
srcPtr += srcSize;
srcSize = srcEnd - srcPtr;
}
/* Decompress:
* Continue while there is more input to read and the frame isn't over.
* If srcPtr == srcEnd then we know that there is no more output left in the
* internal buffer left to flush.
*/
while (srcPtr != srcEnd && ret != 0) {
/* INVARIANT: Any data left in dst has already been written */
size_t dstSize = dstCapacity;
ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
if (LZ4F_isError(ret)) {
printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
goto cleanup;
}
/* Flush output */
if (dstSize != 0){
size_t written = fwrite(dst, 1, dstSize, out);
printf("Writing %zu bytes\n", dstSize);
if (written != dstSize) {
printf("Decompress: Failed to write to file\n");
goto cleanup;
}
}
/* Update input */
srcPtr += srcSize;
srcSize = srcEnd - srcPtr;
}
}
/* Check that there isn't trailing input data after the frame.
* It is valid to have multiple frames in the same file, but this example
* doesn't support it.
*/
ret = fread(src, 1, 1, in);
if (ret != 0 || !feof(in)) {
printf("Decompress: Trailing data left in file after frame\n");
goto cleanup;
}
srcPtr += srcSize;
srcSize = srcEnd - srcPtr;
}
/* Decompress:
* Continue while there is more input to read and the frame isn't over.
* If srcPtr == srcEnd then we know that there is no more output left in the
* internal buffer left to flush.
*/
while (srcPtr != srcEnd && ret != 0) {
/* INVARIANT: Any data left in dst has already been written */
size_t dstSize = dstCapacity;
ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
if (LZ4F_isError(ret)) {
printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
goto cleanup;
}
/* Flush output */
if (dstSize != 0){
size_t written = fwrite(dst, 1, dstSize, out);
printf("Writing %zu bytes\n", dstSize);
if (written != dstSize) {
printf("Decompress: Failed to write to file\n");
goto cleanup;
}
}
/* Update input */
srcPtr += srcSize;
srcSize = srcEnd - srcPtr;
}
}
/* Check that there isn't trailing input data after the frame.
* It is valid to have multiple frames in the same file, but this example
* doesn't support it.
*/
ret = fread(src, 1, 1, in);
if (ret != 0 || !feof(in)) {
printf("Decompress: Trailing data left in file after frame\n");
goto cleanup;
}
cleanup:
free(src);
free(dst);
return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
free(src);
free(dst);
return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
}
int compare(FILE* fp0, FILE* fp1)
{
int result = 0;
int result = 0;
while(0 == result) {
char b0[1024];
char b1[1024];
const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
while(0 == result) {
char b0[1024];
char b1[1024];
const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
result = (int) r0 - (int) r1;
result = (int) r0 - (int) r1;
if (0 == r0 || 0 == r1) {
break;
}
if (0 == result) {
result = memcmp(b0, b1, r0);
}
}
if (0 == r0 || 0 == r1) {
break;
}
if (0 == result) {
result = memcmp(b0, b1, r0);
}
}
return result;
return result;
}
int main(int argc, const char **argv) {
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
/* compress */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
size_t sizeIn = 0;
size_t sizeOut = 0;
size_t ret;
/* compress */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
size_t sizeIn = 0;
size_t sizeOut = 0;
size_t ret;
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
if (ret) {
printf("compress : failed with code %zu\n", ret);
return ret;
}
printf("%s: %zu → %zu bytes, %.1f%%\n",
inpFilename, sizeIn, sizeOut,
(double)sizeOut / sizeIn * 100);
printf("compress : done\n");
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
if (ret) {
printf("compress : failed with code %zu\n", ret);
return (int)ret;
}
printf("%s: %zu → %zu bytes, %.1f%%\n",
inpFilename, sizeIn, sizeOut,
(double)sizeOut / sizeIn * 100);
printf("compress : done\n");
fclose(outFp);
fclose(inpFp);
}
fclose(outFp);
fclose(inpFp);
}
/* decompress */
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
size_t ret;
/* decompress */
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
size_t ret;
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
ret = decompress_file(inpFp, outFp);
if (ret) {
printf("decompress : failed with code %zu\n", ret);
return ret;
}
printf("decompress : done\n");
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
ret = decompress_file(inpFp, outFp);
if (ret) {
printf("decompress : failed with code %zu\n", ret);
return (int)ret;
}
printf("decompress : done\n");
fclose(outFp);
fclose(inpFp);
}
fclose(outFp);
fclose(inpFp);
}
/* verify */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
/* verify */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
fclose(decFp);
fclose(inpFp);
}
fclose(decFp);
fclose(inpFp);
}
return 0;
}

2
examples/printVersion.c
View File

@ -1,5 +1,5 @@
// LZ4 trivial example : print Library version number
// Copyright : Takayuki Matsuoka & Yann Collet
// by Takayuki Matsuoka
#include <stdio.h>

82
examples/simple_buffer.c
View File

@ -8,10 +8,10 @@
*/
/* Includes, for Power! */
#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression.
#include <stdio.h> // For printf()
#include <string.h> // For memcmp()
#include <stdlib.h> // For exit()
#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression.
#include <stdio.h> // For printf()
#include <string.h> // For memcmp()
#include <stdlib.h> // For exit()
/*
* Easy show-error-and-bail function.
@ -28,37 +28,39 @@ void run_screaming(const char *message, const int code) {
*/
int main(void) {
/* Introduction */
// Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start:
// 1) The return codes of LZ4_ functions are important. Read lz4.h if you're unsure what a given code means.
// 2) LZ4 uses char* pointers in all LZ4_ functions. This is baked into the API and probably not going to change. If your
// program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the
// right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
// Below we will have a Compression and Decompression section to demonstrate.
// There are a few important notes before we start:
// 1) The return codes of LZ4_ functions are important.
// Read lz4.h if you're unsure what a given code means.
// 2) LZ4 uses char* pointers in all LZ4_ functions.
// This is baked into the API and probably not going to change.
// If your program uses pointers that are unsigned char*, void*, or otherwise different,
// you may need to do some casting or set the right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
/* Compression */
// We'll store some text into a variable pointed to by *src to be compressed later.
const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
// The compression function needs to know how many bytes of exist. Since we're using a string, we can use strlen() + 1 (for \0).
const size_t src_size = strlen(src) + 1;
const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
// The compression function needs to know how many bytes exist. Since we're using a string, we can use strlen() + 1 (for \0).
const int src_size = (int)(strlen(src) + 1);
// LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
const size_t max_dst_size = LZ4_compressBound(src_size);
const int max_dst_size = LZ4_compressBound(src_size);
// We will use that size for our destination boundary when allocating space.
char *compressed_data = malloc(max_dst_size);
char* compressed_data = malloc(max_dst_size);
if (compressed_data == NULL)
run_screaming("Failed to allocate memory for *compressed_data.", 1);
// That's all the information and preparation LZ4 needs to compress *src into *compressed_data. Invoke LZ4_compress_default now
// with our size values and pointers to our memory locations. Save the return value for error checking.
int return_value = 0;
return_value = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
// That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
// Invoke LZ4_compress_default now with our size values and pointers to our memory locations.
// Save the return value for error checking.
const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
// Check return_value to determine what happened.
if (return_value < 0)
run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value);
if (return_value == 0)
if (compressed_data_size < 0)
run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", compressed_data_size);
if (compressed_data_size == 0)
run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1);
if (return_value > 0)
if (compressed_data_size > 0)
printf("We successfully compressed some data!\n");
// Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to
// realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept.
const size_t compressed_data_size = return_value;
// Not only does a positive return_value mean success, the value returned == the number of bytes required.
// You can use this to realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept.
compressed_data = (char *)realloc(compressed_data, compressed_data_size);
if (compressed_data == NULL)
run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1);
@ -66,25 +68,27 @@ int main(void) {
/* Decompression */
// Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a
// *new_src location of size src_size since we know that value.
char *new_src = malloc(src_size);
if (new_src == NULL)
run_screaming("Failed to allocate memory for *new_src.", 1);
// The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src
// memory location is, and how large the new_src (uncompressed) output will be. Again, save the return_value.
return_value = LZ4_decompress_safe(compressed_data, new_src, compressed_data_size, src_size);
if (return_value < 0)
run_screaming("A negative result from LZ4_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value);
if (return_value == 0)
char* const regen_buffer = malloc(src_size);
if (regen_buffer == NULL)
run_screaming("Failed to allocate memory for *regen_buffer.", 1);
// The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is,
// where the regen_buffer memory location is, and how large regen_buffer (uncompressed) output will be.
// Again, save the return_value.
const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressed_data_size, src_size);
free(compressed_data); /* no longer useful */
if (decompressed_size < 0)
run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", decompressed_size);
if (decompressed_size == 0)
run_screaming("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.", 1);
if (return_value > 0)
if (decompressed_size > 0)
printf("We successfully decompressed some data!\n");
// Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data
// stream. I'm not sure there's ever a time you'll need to know this in most cases...
// Not only does a positive return value mean success,
// value returned == number of bytes regenerated from compressed_data stream.
/* Validation */
// We should be able to compare our original *src with our *new_src and be byte-for-byte identical.
if (memcmp(src, new_src, src_size) != 0)
if (memcmp(src, regen_buffer, src_size) != 0)
run_screaming("Validation failed. *src and *new_src are not identical.", 1);
printf("Validation done. The string we ended up with is:\n%s\n", new_src);
printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer);
return 0;
}

127
lib/Makefile
View File

@ -3,6 +3,8 @@
# Copyright (C) Yann Collet 2011-2016
# All rights reserved.
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
# BSD license
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
@ -38,22 +40,19 @@ LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCR
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
BUILD_STATIC:= yes
BUILD_STATIC:=yes
DESTDIR?=
PREFIX ?= /usr/local
CPPFLAGS= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wundef -Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
LIBDIR?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
SRCFILES := $(sort $(wildcard *.c))
# OS X linker doesn't support -soname, and use different extension
@ -62,7 +61,7 @@ ifeq ($(shell uname), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
@ -70,73 +69,109 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
LIBLZ4 = liblz4.$(SHARED_EXT_VER)
.PHONY: default
default: lib-release
lib-release: DEBUGFLAGS :=
lib-release: lib
lib: liblz4.a liblz4
all: lib
all32: CFLAGS+=-m32
all32: all
lib: liblz4.a liblz4
lib-release: CFLAGS := -O3
lib-release: lib
liblz4.a: *.c
ifeq ($(BUILD_STATIC),yes)
liblz4.a: $(SRCFILES)
ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
@$(CC) $(FLAGS) -c $^
@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
@$(AR) rcs $@ *.o
endif
liblz4: *.c
$(LIBLZ4): $(SRCFILES)
@echo compiling dynamic library $(LIBVER)
ifneq (,$(filter Windows%,$(OS)))
@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib
else
@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
@echo creating versioned links
@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
@ln -sf $@ liblz4.$(SHARED_EXT)
endif
liblz4: $(LIBLZ4)
clean:
@$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib
@$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib
@$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
@echo Cleaning library completed
#------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
# support both lower and uppercase (BSD), use uppercase in script
prefix ?= /usr/local
PREFIX ?= $(prefix)
exec_prefix ?= $(PREFIX)
libdir ?= $(exec_prefix)/lib
LIBDIR ?= $(libdir)
includedir ?= $(PREFIX)/include
INCLUDEDIR ?= $(includedir)
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
endif
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
endif
INSTALL_PROGRAM ?= $(INSTALL)
INSTALL_DATA ?= $(INSTALL) -m 644
liblz4.pc: liblz4.pc.in Makefile
@echo creating pkgconfig
@sed -e 's|@PREFIX@|$(PREFIX)|' \
-e 's|@LIBDIR@|$(LIBDIR)|' \
-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-e 's|@VERSION@|$(LIBVER)|' \
$< >$@
-e 's|@LIBDIR@|$(LIBDIR)|' \
-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-e 's|@VERSION@|$(LIBVER)|' \
$< >$@
install: lib liblz4.pc
@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
@install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
@$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
@echo Installing libraries
ifeq ($(BUILD_STATIC),yes)
@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
endif
@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
@install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
ifeq ($(BUILD_STATIC),yes)
@install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
endif
@install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
@echo lz4 static and shared libraries installed
@echo Installing headers in $(INCLUDEDIR)
@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
@echo lz4 libraries installed
uninstall:
@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h

58
lib/README.md
View File

@ -1,44 +1,43 @@
LZ4 - Library Files
================================
The directory contains many files, but depending on project's objectives,
The `/lib` directory contains many files, but depending on project's objectives,
not all of them are necessary.
#### Minimal LZ4 build
The minimum required is **`lz4.c`** and **`lz4.h`**,
which will provide the fast compression and decompression algorithm.
which provides the fast compression and decompression algorithm.
They generate and decode data using [LZ4 block format].
#### The High Compression variant of LZ4
#### High Compression variant
For more compression at the cost of compression speed,
the High Compression variant **lz4hc** is available.
It's necessary to add **`lz4hc.c`** and **`lz4hc.h`**.
The variant still depends on regular `lz4` source files.
In particular, the decompression is still provided by `lz4.c`.
For more compression ratio at the cost of compression speed,
the High Compression variant called **lz4hc** is available.
Add files **`lz4hc.c`**, **`lz4hc.h`** and **`lz4opt.h`**.
The variant still depends on regular `lib/lz4.*` source files.
#### Compatibility issues
#### Frame variant, for interoperability
In order to produce files or streams compatible with `lz4` command line utility,
In order to produce compressed data compatible with `lz4` command line utility,
it's necessary to encode lz4-compressed blocks using the [official interoperable frame format].
This format is generated and decoded automatically by the **lz4frame** library.
In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**,
which provides error detection.
(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace.
This is what `liblz4` does, to avoid symbol duplication
in case a user program would link to several libraries containing xxhash symbols.)
Its public API is described in `lib/lz4frame.h`.
In order to work properly, lz4frame needs all other modules present in `/lib`,
including, lz4 and lz4hc, and also **xxhash**.
So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
#### Advanced API
#### Advanced / Experimental API
A more complex `lz4frame_static.h` is also provided.
It contains definitions which are not guaranteed to remain stable within future versions.
It must be used with static linking ***only***.
A complex API defined in `lz4frame_static.h` contains definitions
which are not guaranteed to remain stable in future versions.
As a consequence, it must be used with static linking ***only***.
#### Using MinGW+MSYS to create DLL
#### Windows : using MinGW+MSYS to create DLL
DLL can be created using MinGW+MSYS with the `make liblz4` command.
This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`.
@ -47,27 +46,28 @@ The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library
`dll\liblz4.dll` are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses LZ4 consists of a single `test-dll.c`
file it should be compiled with "liblz4.lib". For example:
file it should be linked with `dll\liblz4.dll`. For example:
```
gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll
```
The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
#### Miscellaneous
#### Miscellaneous
Other files present in the directory are not source code. There are :
- LICENSE : contains the BSD license text
- Makefile : script to compile or install lz4 library (static or dynamic)
- liblz4.pc.in : for pkg-config (make install)
- README.md : this file
- `LICENSE` : contains the BSD license text
- `Makefile` : `make` script to compile and install lz4 library (static and dynamic)
- `liblz4.pc.in` : for `pkg-config` (used in `make install`)
- `README.md` : this file
[official interoperable frame format]: ../doc/lz4_Frame_format.md
[LZ4 block format]: ../doc/lz4_Block_format.md
#### License
#### License
All source material within __lib__ directory are BSD 2-Clause licensed.
See [LICENSE](LICENSE) for details.
The license is also repeated at the top of each source file.
The license is also reminded at the top of each source file.

2
lib/dll/example/README.md
View File

@ -39,7 +39,7 @@ The header files from `include\` and the dynamic library `dll\liblz4.dll`
are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses LZ4 consists of a single `test-dll.c`
file it should be compiled with "liblz4.dll". For example:
file it should be linked with `dll\liblz4.dll`. For example:
```
gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll
```

240
lib/lz4.c
View File

@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -37,12 +37,12 @@
* Tuning parameters
**************************************/
/*
* HEAPMODE :
* LZ4_HEAPMODE :
* Select how default compression functions will allocate memory for their hash table,
* in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
*/
#ifndef HEAPMODE
# define HEAPMODE 0
#ifndef LZ4_HEAPMODE
# define LZ4_HEAPMODE 0
#endif
/*
@ -63,16 +63,15 @@
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets which generate assembly depending on alignment.
* It can generate buggy code on targets which assembly generation depends on alignment.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define LZ4_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# elif defined(__INTEL_COMPILER) || defined(__GNUC__)
# define LZ4_FORCE_MEMORY_ACCESS 1
# endif
#endif
@ -86,6 +85,7 @@
#endif
/*-************************************
* Dependency
**************************************/
@ -97,20 +97,49 @@
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h>
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
#else
# if defined(__GNUC__) || defined(__clang__)
# define FORCE_INLINE static inline __attribute__((always_inline))
# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define FORCE_INLINE static inline
# else
# define FORCE_INLINE static
# endif
#endif /* _MSC_VER */
#ifndef LZ4_FORCE_INLINE
# ifdef _MSC_VER /* Visual Studio */
# define LZ4_FORCE_INLINE static __forceinline
# else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define LZ4_FORCE_INLINE static inline __attribute__((always_inline))
# else
# define LZ4_FORCE_INLINE static inline
# endif
# else
# define LZ4_FORCE_INLINE static
# endif /* __STDC_VERSION__ */
# endif /* _MSC_VER */
#endif /* LZ4_FORCE_INLINE */
/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE
* Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy,
* together with a simple 8-byte copy loop as a fall-back path.
* However, this optimization hurts the decompression speed by >30%,
* because the execution does not go to the optimized loop
* for typical compressible data, and all of the preamble checks
* before going to the fall-back path become useless overhead.
* This optimization happens only with the -O3 flag, and -O2 generates
* a simple 8-byte copy loop.
* With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy
* functions are annotated with __attribute__((optimize("O2"))),
* and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute
* of LZ4_wildCopy does not affect the compression speed.
*/
#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__)
# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2")))
# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE
#else
# define LZ4_FORCE_O2_GCC_PPC64LE
# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static
#endif
#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
@ -247,7 +276,8 @@ static void LZ4_copy8(void* dst, const void* src)
}
/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
LZ4_FORCE_O2_INLINE_GCC_PPC64LE
void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
{
BYTE* d = (BYTE*)dstPtr;
const BYTE* s = (const BYTE*)srcPtr;
@ -281,15 +311,36 @@ static const int LZ4_minLength = (MFLIMIT+1);
/*-************************************
* Common Utils
* Error detection
**************************************/
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
# include <assert.h>
#else
# ifndef assert
# define assert(condition) ((void)0)
# endif
#endif
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
# include <stdio.h>
static int g_debuglog_enable = 1;
# define DEBUGLOG(l, ...) { \
if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \
} }
#else
# define DEBUGLOG(l, ...) {} /* disabled */
#endif
/*-************************************
* Common functions
**************************************/
static unsigned LZ4_NbCommonBytes (register reg_t val)
static unsigned LZ4_NbCommonBytes (reg_t val)
{
if (LZ4_isLittleEndian()) {
if (sizeof(val)==8) {
@ -300,7 +351,14 @@ static unsigned LZ4_NbCommonBytes (register reg_t val)
# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
0, 3, 1, 3, 1, 4, 2, 7,
0, 2, 3, 6, 1, 5, 3, 5,
1, 3, 4, 4, 2, 5, 6, 7,
7, 0, 1, 2, 3, 3, 4, 6,
2, 6, 5, 5, 3, 4, 5, 6,
7, 1, 2, 4, 6, 4, 4, 5,
7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
} else /* 32 bits */ {
@ -311,12 +369,15 @@ static unsigned LZ4_NbCommonBytes (register reg_t val)
# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
3, 2, 2, 1, 3, 2, 0, 1,
3, 3, 1, 2, 2, 2, 2, 0,
3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
} else /* Big Endian CPU */ {
if (sizeof(val)==8) {
if (sizeof(val)==8) { /* 64-bits */
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
@ -324,8 +385,11 @@ static unsigned LZ4_NbCommonBytes (register reg_t val)
# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll((U64)val) >> 3);
# else
static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits.
Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
Note that this code path is never triggered in 32-bits mode. */
unsigned r;
if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
@ -348,11 +412,20 @@ static unsigned LZ4_NbCommonBytes (register reg_t val)
}
#define STEPSIZE sizeof(reg_t)
static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
LZ4_FORCE_INLINE
unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while (likely(pIn<pInLimit-(STEPSIZE-1))) {
if (likely(pIn < pInLimit-(STEPSIZE-1))) {
reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
if (!diff) {
pIn+=STEPSIZE; pMatch+=STEPSIZE;
} else {
return LZ4_NbCommonBytes(diff);
} }
while (likely(pIn < pInLimit-(STEPSIZE-1))) {
reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; }
pIn += LZ4_NbCommonBytes(diff);
@ -391,6 +464,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
* Local Utils
**************************************/
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
@ -417,7 +491,7 @@ static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
{
if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
return LZ4_hash4(LZ4_read32(p), tableType);
@ -433,7 +507,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
}
}
FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
U32 const h = LZ4_hashPosition(p, tableType);
LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
@ -446,7 +520,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab
{ const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
}
FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
U32 const h = LZ4_hashPosition(p, tableType);
return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@ -455,7 +529,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy
/** LZ4_compress_generic() :
inlined, to ensure branches are decided at compilation time */
FORCE_INLINE int LZ4_compress_generic(
LZ4_FORCE_INLINE int LZ4_compress_generic(
LZ4_stream_t_internal* const cctx,
const char* const source,
char* const dest,
@ -597,7 +671,11 @@ _next_match:
*token += ML_MASK;
matchCode -= ML_MASK;
LZ4_write32(op, 0xFFFFFFFF);
while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255;
while (matchCode >= 4*255) {
op+=4;
LZ4_write32(op, 0xFFFFFFFF);
matchCode -= 4*255;
}
op += matchCode / 255;
*op++ = (BYTE)(matchCode % 255);
} else
@ -677,7 +755,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
#if (HEAPMODE)
#if (LZ4_HEAPMODE)
void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctx;
@ -686,7 +764,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
#if (HEAPMODE)
#if (LZ4_HEAPMODE)
FREEMEM(ctxPtr);
#endif
return result;
@ -890,7 +968,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
#if (HEAPMODE)
#if (LZ4_HEAPMODE)
LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctxBody;
@ -899,7 +977,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
#if (HEAPMODE)
#if (LZ4_HEAPMODE)
FREEMEM(ctx);
#endif
return result;
@ -921,11 +999,13 @@ LZ4_stream_t* LZ4_createStream(void)
void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
{
DEBUGLOG(4, "LZ4_resetStream");
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
}
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
if (!LZ4_stream) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return (0);
}
@ -1080,47 +1160,46 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
* Decompression functions
*******************************/
/*! LZ4_decompress_generic() :
* This generic decompression function cover all use cases.
* It shall be instantiated several times, using different sets of directives
* Note that it is important this generic function is really inlined,
* This generic decompression function covers all use cases.
* It shall be instantiated several times, using different sets of directives.
* Note that it is important for performance that this function really get inlined,
* in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int LZ4_decompress_generic(
const char* const source,
char* const dest,
int inputSize,
int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */
LZ4_FORCE_O2_GCC_PPC64LE
LZ4_FORCE_INLINE int LZ4_decompress_generic(
const char* const src,
char* const dst,
int srcSize,
int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */
int endOnInput, /* endOnOutputSize, endOnInputSize */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest when no prefix */
const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize /* note : = 0 if noDict */
)
{
/* Local Variables */
const BYTE* ip = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
BYTE* op = (BYTE*) dest;
BYTE* op = (BYTE*) dst;
BYTE* const oend = op + outputSize;
BYTE* cpy;
BYTE* oexit = op + targetOutputSize;
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};
const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4};
const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};
const int safeDecode = (endOnInput==endOnInputSize);
const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
/* Special cases */
if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */
if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
/* Main Loop : decode sequences */
@ -1129,8 +1208,27 @@ FORCE_INLINE int LZ4_decompress_generic(
const BYTE* match;
size_t offset;
/* get literal length */
unsigned const token = *ip++;
/* shortcut for common case :
* in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
* this shortcut was tested on x86 and x64, where it improves decoding speed.
* it has not yet been benchmarked on ARM, Power, mips, etc. */
if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend)
& (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
& ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
size_t const ll = token >> ML_BITS;
size_t const off = LZ4_readLE16(ip+ll);
const BYTE* const matchPtr = op + ll - off; /* pointer underflow risk ? */
if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
size_t const ml = (token & ML_MASK) + MINMATCH;
memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
memcpy(op, matchPtr, 18); op += ml;
continue;
}
}
/* decode literal length */
if ((length=(token>>ML_BITS)) == RUN_MASK) {
unsigned s;
do {
@ -1164,7 +1262,7 @@ FORCE_INLINE int LZ4_decompress_generic(
/* get offset */
offset = LZ4_readLE16(ip); ip+=2;
match = op - offset;
if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */
if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */
/* get matchlength */
@ -1208,14 +1306,13 @@ FORCE_INLINE int LZ4_decompress_generic(
/* copy match within block */
cpy = op + length;
if (unlikely(offset<8)) {
const int dec64 = dec64table[offset];
op[0] = match[0];
op[1] = match[1];
op[2] = match[2];
op[3] = match[3];
match += dec32table[offset];
match += inc32table[offset];
memcpy(op+4, match, 4);
match -= dec64;
match -= dec64table[offset];
} else { LZ4_copy8(op, match); match+=8; }
op += 8;
@ -1232,31 +1329,34 @@ FORCE_INLINE int LZ4_decompress_generic(
LZ4_copy8(op, match);
if (length>16) LZ4_wildCopy(op+8, match+8, cpy);
}
op=cpy; /* correction */
op = cpy; /* correction */
}
/* end of decoding */
if (endOnInput)
return (int) (((char*)op)-dest); /* Nb of output bytes decoded */
return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
else
return (int) (((const char*)ip)-source); /* Nb of input bytes read */
return (int) (((const char*)ip)-src); /* Nb of input bytes read */
/* Overflow error detected */
_output_error:
return (int) (-(((const char*)ip)-source))-1;
return (int) (-(((const char*)ip)-src))-1;
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
@ -1265,11 +1365,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
/*===== streaming decompression functions =====*/
/*
* If you prefer dynamic allocation methods,
* LZ4_createStreamDecode()
* provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
*/
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
@ -1278,6 +1373,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void)
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
{
if (!LZ4_stream) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return 0;
}
@ -1306,6 +1402,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
If it's not possible, save the relevant part of decoded data into a safe buffer,
and indicate where it stands using LZ4_setStreamDecode()
*/
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
@ -1332,6 +1429,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
return result;
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
{
LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
@ -1366,7 +1464,8 @@ Advanced decoding functions :
the dictionary must be explicitly provided within parameters
*/
FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
LZ4_FORCE_O2_GCC_PPC64LE
LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
{
if (dictSize==0)
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
@ -1378,17 +1477,20 @@ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
}
/* debug function */
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);

256
lib/lz4.h
View File

@ -1,7 +1,7 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
* Copyright (C) 2011-2016, Yann Collet.
* Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -32,13 +32,13 @@
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
#ifndef LZ4_H_2983827168210
#define LZ4_H_2983827168210
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef LZ4_H_2983827168210
#define LZ4_H_2983827168210
/* --- Dependency --- */
#include <stddef.h> /* size_t */
@ -72,20 +72,28 @@ extern "C" {
/*
* LZ4_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
* LZ4LIB_VISIBILITY :
* Control library symbols visibility.
*/
#ifndef LZ4LIB_VISIBILITY
# if defined(__GNUC__) && (__GNUC__ >= 4)
# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))
# else
# define LZ4LIB_VISIBILITY
# endif
#endif
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
# define LZ4LIB_API __declspec(dllexport)
# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define LZ4LIB_API
# define LZ4LIB_API LZ4LIB_VISIBILITY
#endif
/*========== Version =========== */
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@ -94,8 +102,8 @@ extern "C" {
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
LZ4LIB_API int LZ4_versionNumber (void);
LZ4LIB_API const char* LZ4_versionString (void);
LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */
LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */
/*-************************************
@ -108,37 +116,38 @@ LZ4LIB_API const char* LZ4_versionString (void);
* Reduced memory usage can improve speed, due to cache effect
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#define LZ4_MEMORY_USAGE 14
#ifndef LZ4_MEMORY_USAGE
# define LZ4_MEMORY_USAGE 14
#endif
/*-************************************
* Simple Functions
**************************************/
/*! LZ4_compress_default() :
Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
Compresses 'srcSize' bytes from buffer 'src'
into already allocated 'dst' buffer of size 'dstCapacity'.
Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
If the function cannot compress 'src' into a limited 'dst' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails */
LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
As a consequence, 'dst' content is not valid.
This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
srcSize : supported max value is LZ4_MAX_INPUT_VALUE
dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
or 0 if compression fails */
LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
/*! LZ4_decompress_safe() :
compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
compressedSize : is the exact complete size of the compressed block.
dstCapacity : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
If destination buffer is not large enough, decoding will stop and output an error code (negative value).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
*/
LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
/*-************************************
@ -167,7 +176,7 @@ LZ4_compress_fast() :
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
*/
LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*!
@ -178,49 +187,49 @@ LZ4_compress_fast_extState() :
Then, provide it as 'void* state' to compression function.
*/
LZ4LIB_API int LZ4_sizeofState(void);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*!
LZ4_compress_destSize() :
Reverse the logic, by compressing as much data as possible from 'source' buffer
into already allocated buffer 'dest' of size 'targetDestSize'.
This function either compresses the entire 'source' content into 'dest' if it's large enough,
or fill 'dest' buffer completely with as much data as possible from 'source'.
*sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
New value is necessarily <= old value.
return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
or 0 if compression fails
Reverse the logic : compresses as much data as possible from 'src' buffer
into already allocated buffer 'dst' of size 'targetDestSize'.
This function either compresses the entire 'src' content into 'dst' if it's large enough,
or fill 'dst' buffer completely with as much data as possible from 'src'.
*srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
New value is necessarily <= old value.
return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
or 0 if compression fails
*/
LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
/*!
LZ4_decompress_fast() :
originalSize : is the original and therefore uncompressed size
LZ4_decompress_fast() : (unsafe!!)
originalSize : is the original uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size)
If the source stream is detected malformed, the function will stop decoding and return a negative result.
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
note : This function fully respect memory boundaries for properly formed compressed data.
Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
note : This function respects memory boundaries for *properly formed* compressed data.
It is a bit faster than LZ4_decompress_safe().
However, it does not provide any protection against intentionally modified data stream (malicious input).
Use this function in trusted environment only (data to decode comes from a trusted source).
*/
LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
/*!
LZ4_decompress_safe_partial() :
This function decompress a compressed block of size 'compressedSize' at position 'source'
into destination buffer 'dest' of size 'maxDecompressedSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
This function decompress a compressed block of size 'srcSize' at position 'src'
into destination buffer 'dst' of size 'dstCapacity'.
The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
@return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
*/
LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
/*-*********************************************
@ -237,33 +246,38 @@ LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
/*! LZ4_resetStream() :
* An LZ4_stream_t structure can be allocated once and re-used multiple times.
* Use this function to init an allocated `LZ4_stream_t` structure and start a new compression.
* Use this function to start compressing a new stream.
*/
LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
/*! LZ4_loadDict() :
* Use this function to load a static dictionary into LZ4_stream.
* Use this function to load a static dictionary into LZ4_stream_t.
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
* Loading a size of 0 is allowed.
* Return : dictionary size, in bytes (necessarily <= 64 KB)
* Loading a size of 0 is allowed, and is the same as reset.
* @return : dictionary size, in bytes (necessarily <= 64 KB)
*/
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! LZ4_compress_fast_continue() :
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
* Important : Previous data blocks are assumed to still be present and unmodified !
* Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
* 'dst' buffer must be already allocated.
* If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
*
* Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
* Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
* Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
*
* @return : size of compressed block
* or 0 if there is an error (typically, compressed data cannot fit into 'dst')
* After an error, the stream status is invalid, it can only be reset or freed.
*/
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
* If previously compressed data block is not guaranteed to remain available at its current memory location,
* save it into a safer place (char* safeBuffer).
* Note : you don't need to call LZ4_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
* Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
* @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
*/
LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
@ -274,36 +288,41 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict
************************************************/
typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */
/* creation / destruction of streaming decompression tracking structure */
/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
* creation / destruction of streaming decompression tracking structure.
* A tracking structure can be re-used multiple times sequentially. */
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
/*! LZ4_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
* Setting a size of 0 is allowed (same effect as reset).
* @return : 1 if OK, 0 if error
* An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
* Use this function to start decompression of a new stream of blocks.
* A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
* @return : 1 if OK, 0 if error
*/
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
/*!
LZ4_decompress_*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ4_setStreamDecode()
/*! LZ4_decompress_*_continue() :
* These decoding functions allow decompression of consecutive blocks in "streaming" mode.
* A block is an unsplittable entity, it must be presented entirely to a decompression function.
* Decompression functions only accept one block at a time.
* Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
*
* Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
* - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
* In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
* - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
* maxBlockSize is implementation dependent. It's the maximum size of any single block.
* In which case, encoding and decoding buffers do not need to be synchronized,
* and encoding ring buffer can have any size, including small ones ( < 64 KB).
* - _At least_ 64 KB + 8 bytes + maxBlockSize.
* In which case, encoding and decoding buffers do not need to be synchronized,
* and encoding ring buffer can have any size, including larger than decoding buffer.
* Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
* and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
*/
LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
/*! LZ4_decompress_*_usingDict() :
@ -311,8 +330,8 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod
* a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
* They are stand-alone, and don't need an LZ4_streamDecode_t structure.
*/
LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
/*^**********************************************
@ -374,7 +393,7 @@ typedef struct {
* init this structure before first use.
* note : only use in association with static linking !
* this definition is not API/ABI safe,
* and may change in a future version !
* it may change in a future version !
*/
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
@ -400,11 +419,12 @@ union LZ4_streamDecode_u {
} ; /* previously typedef'd to LZ4_streamDecode_t */
/*=************************************
/*-************************************
* Obsolete Functions
**************************************/
/* Deprecation warnings */
/* Should these warnings be a problem,
/*! Deprecation warnings
Should deprecation warnings be a problem,
it is generally possible to disable them,
typically with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual.
@ -413,9 +433,11 @@ union LZ4_streamDecode_u {
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
#else
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# elif (LZ4_GCC_VERSION >= 405)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
@ -428,36 +450,30 @@ union LZ4_streamDecode_u {
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
/* Obsolete compression functions */
LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize);
LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete decompression functions */
/* These function names are completely deprecated and must no longer be used.
They are only provided in lz4.c for compatibility with older programs.
- LZ4_uncompress is the same as LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
These function prototypes are now disabled; uncomment them only if you really need them.
It is highly recommended to stop using these prototypes and migrate to maintained ones */
/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/* Obsolete streaming functions; use new streaming interface whenever possible */
LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
#endif /* LZ4_H_2983827168210 */
#if defined (__cplusplus)
}
#endif
#endif /* LZ4_H_2983827168210 */

1337
lib/lz4frame.c
View File

File diff suppressed because it is too large Load Diff

281
lib/lz4frame.h
View File

@ -1,7 +1,7 @@
/*
LZ4 auto-framing library
Header File
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@ -48,49 +48,68 @@ extern "C" {
/* --- Dependency --- */
#include <stddef.h> /* size_t */
/*-***************************************************************
* Compiler specifics
*****************************************************************/
/*!
* LZ4_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
/**
Introduction
lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
lz4frame.h provides frame compression functions that take care
of encoding standard metadata alongside LZ4-compressed blocks.
*/
/*-***************************************************************
* Compiler specifics
*****************************************************************/
/* LZ4_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
* LZ4FLIB_API :
* Control library symbols visibility.
*/
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
# define LZ4FLIB_API __declspec(dllexport)
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
# define LZ4FLIB_API __declspec(dllimport)
#elif defined(__GNUC__) && (__GNUC__ >= 4)
# define LZ4FLIB_API __attribute__ ((__visibility__ ("default")))
#else
# define LZ4FLIB_API
#endif
#if defined(_MSC_VER)
# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */
#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
# define LZ4F_DEPRECATE(x) x __attribute__((deprecated))
#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS
# define LZ4F_DEPRECATE(x) x
#else
# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */
# if defined(_MSC_VER)
# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */
# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
# define LZ4F_DEPRECATE(x) x __attribute__((deprecated))
# else
# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */
# endif
#endif
/*-************************************
* Error management
**************************************/
* Error management
**************************************/
typedef size_t LZ4F_errorCode_t;
LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code);
LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */
LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */
LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */
/*-************************************
* Frame compression types
**************************************/
/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */
#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
* Frame compression types
**************************************/
/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */
#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
#else
# define LZ4F_OBSOLETE_ENUM(x)
#endif
/* The larger the block size, the (slightly) better the compression ratio,
* though there are diminishing returns.
* Larger blocks also increase memory usage on both compression and decompression sides. */
typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
@ -103,6 +122,9 @@ typedef enum {
LZ4F_OBSOLETE_ENUM(max4MB)
} LZ4F_blockSizeID_t;
/* Linked blocks sharply reduce inefficiencies when using small blocks,
* they compress better.
* However, some LZ4 decoders are only compatible with independent blocks */
typedef enum {
LZ4F_blockLinked=0,
LZ4F_blockIndependent
@ -117,64 +139,71 @@ typedef enum {
LZ4F_OBSOLETE_ENUM(contentChecksumEnabled)
} LZ4F_contentChecksum_t;
typedef enum {
LZ4F_noBlockChecksum=0,
LZ4F_blockChecksumEnabled
} LZ4F_blockChecksum_t;
typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
typedef LZ4F_frameType_t frameType_t;
typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
/* LZ4F_frameInfo_t :
* makes it possible to supply detailed frame parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
/*! LZ4F_frameInfo_t :
* makes it possible to set or read frame parameters.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* For all fields, 0 sets it to default value */
typedef struct {
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */
LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */
} LZ4F_frameInfo_t;
/* LZ4F_preferences_t :
* makes it possible to supply detailed compression parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
/*! LZ4F_preferences_t :
* makes it possible to supply detailed compression parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */
int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
LZ4FLIB_API int LZ4F_compressionLevel_max(void);
/*-*********************************
* Simple compression function
***********************************/
/*!LZ4F_compressFrameBound() :
* Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
* Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
/*! LZ4F_compressFrameBound() :
* Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
* Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
/*!LZ4F_compressFrame() :
* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
* An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
* This value is supplied by LZ4F_compressFrameBound().
* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
/*! LZ4F_compressFrame() :
* Compress an entire srcBuffer into a valid LZ4 frame.
* dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError())
*/
LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
const LZ4F_preferences_t* preferencesPtr);
/*-***********************************
@ -188,55 +217,57 @@ typedef struct {
unsigned reserved[3];
} LZ4F_compressOptions_t;
/* Resource Management */
/*--- Resource Management ---*/
#define LZ4F_VERSION 100
LZ4FLIB_API unsigned LZ4F_getVersion(void);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/* LZ4F_createCompressionContext() :
/*! LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
* The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
* The function will provide a pointer to a fully allocated LZ4F_cctx object.
* If @return != zero, there was an error during context creation.
* Object can release its memory using LZ4F_freeCompressionContext();
*/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/* Compression */
/*---- Compression ----*/
LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes.
#define LZ4F_HEADER_SIZE_MAX 19
/*! LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
* `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
* @return : number of bytes written into dstBuffer for the header
* or an error code (which can be tested using LZ4F_isError())
*/
LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
void* dstBuffer, size_t dstCapacity,
const LZ4F_preferences_t* prefsPtr);
LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
* Different preferences can produce different results.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
/*! LZ4F_compressBound() :
* Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
* prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
* Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
* When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressUpdate() :
/*! LZ4F_compressUpdate() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
* This value is provided by using LZ4F_compressBound().
* An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
* This value is provided by LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
* LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
* `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
* `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_flush() :
/*! LZ4F_flush() :
* When data must be generated and sent immediately, without waiting for a block to be completely filled,
* it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
* `dstCapacity` must be large enough to ensure the operation will be successful.
@ -244,17 +275,18 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci
* @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressEnd() :
* To properly finish the compressed frame, invoke LZ4F_compressEnd().
* It will flush whatever data remained within `cctx` (like LZ4_flush())
* and properly finalize the frame, with an endMark and a checksum.
* `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
/*! LZ4F_compressEnd() :
* To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
* It will flush whatever data remained within `cctx` (like LZ4_flush())
* and properly finalize the frame, with an endMark and a checksum.
* `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
* @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* or an error code if it fails (which can be tested using LZ4F_isError())
* A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
* A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
*/
LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
/*-*********************************
@ -264,66 +296,83 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */
typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
unsigned reserved[3];
unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
unsigned reserved[3]; /* must be set to zero for forward compatibility */
} LZ4F_decompressOptions_t;
/* Resource management */
/*!LZ4F_createDecompressionContext() :
* Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
* The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
* Create an LZ4F_dctx object, to track all decompression operations.
* The version provided MUST be LZ4F_VERSION.
* The function provides a pointer to an allocated and initialized LZ4F_dctx object.
* The result is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
* The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
/*====== Decompression ======*/
/*-***********************************
* Streaming decompression functions
*************************************/
/*!LZ4F_getFrameInfo() :
* This function decodes frame header information (such as max blockSize, frame checksum, etc.).
* Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
* A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
* The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
* Decompression must resume from this point (srcBuffer + *srcSizePtr).
* Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
* Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, dictID, etc.).
* Its usage is optional.
* Extracted information is typically useful for allocation and dictionary.
* This function works in 2 situations :
* - At the beginning of a new frame, in which case
* it will decode information from `srcBuffer`, starting the decoding process.
* Input size must be large enough to successfully decode the entire frame header.
* Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
* It's allowed to provide more input data than this minimum.
* - After decoding has been started.
* In which case, no input is read, frame parameters are extracted from dctx.
* - If decoding has barely started, but not yet extracted information from header,
* LZ4F_getFrameInfo() will fail.
* The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
* Decompression must resume from (srcBuffer + *srcSizePtr).
* @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
* or an error code which can be tested using LZ4F_isError()
* (typically, when there is not enough src bytes to fully decode the frame header)
* or an error code which can be tested using LZ4F_isError().
* note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/*!LZ4F_decompress() :
* Call this function repetitively to regenerate data compressed within `srcBuffer`.
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate compressed data from `srcBuffer`.
* The function will read up to *srcSizePtr bytes from srcBuffer,
* and decompress data into dstBuffer, of capacity *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
* The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
* The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
*
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* Number of bytes read can be < number of bytes provided, meaning there is some more data to decode.
* It typically happens when dstBuffer is not large enough to contain all decoded data.
* Remaining data will have to be presented again in a subsequent invocation.
* The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
* Unconsumed source data must be presented again in subsequent invocations.
*
* `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
* `dstBuffer` can be changed at will between each consecutive function invocation.
* `dstBuffer` can freely change between each consecutive function invocation.
* `dstBuffer` content will be overwritten.
*
* @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
* Schematically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
* This is just a hint though, it's always possible to provide any srcSize.
* When a frame is fully decoded, @return will be 0 (no more data expected).
* If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
* @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
* Schematically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
* This is just a hint though, it's always possible to provide any srcSize.
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
* When a frame is fully decoded, @return will be 0 (no more data expected).
* When provided with more bytes than necessary to decode a frame,
* LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0.
*
* If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
* After a decompression error, the `dctx` context is not resumable.
* Use LZ4F_resetDecompressionContext() to return to clean state.
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
@ -331,6 +380,14 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
const LZ4F_decompressOptions_t* dOptPtr);
/*! LZ4F_resetDecompressionContext() : added in v1.8.0
* In case of an error, the context is left in "undefined" state.
* In which case, it's necessary to reset it, before re-using it.
* This method can also be used to abruptly stop any unfinished decompression,
* and start a new one using same context resources. */
LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */
#if defined (__cplusplus)
}

102
lib/lz4frame_static.h
View File

@ -43,7 +43,15 @@ extern "C" {
/* lz4frame_static.h should be used solely in the context of static linking.
* It contains definitions which are not stable and may change in the future.
* Never use it in the context of DLL linking.
* */
*
* Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at
* your own risk.
*/
#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
#define LZ4FLIB_STATIC_API LZ4FLIB_API
#else
#define LZ4FLIB_STATIC_API
#endif
/* --- Dependency --- */
@ -52,27 +60,91 @@ extern "C" {
/* --- Error List --- */
#define LZ4F_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(OK_NoError) \
ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) \
ITEM(ERROR_blockMode_invalid) \
ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_headerVersion_wrong) \
ITEM(ERROR_blockChecksum_invalid) \
ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcSize_tooLarge) \
ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameHeader_incomplete) \
ITEM(ERROR_frameType_unknown) \
ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_headerChecksum_invalid) \
ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_frameDecoding_alreadyStarted) \
ITEM(ERROR_maxCode)
#define LZ4F_DISABLE_OLD_ENUMS /* comment to enable deprecated enums */
#ifndef LZ4F_DISABLE_OLD_ENUMS
# define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM,
#else
# define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
#endif
typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
/* enum list is exposed, to handle specific errors */
typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
/**********************************
* Bulk processing dictionary API
*********************************/
typedef struct LZ4F_CDict_s LZ4F_CDict;
/*! LZ4_createCDict() :
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
* `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict);
/*! LZ4_compressFrame_usingCDict() :
* Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
* If cdict==NULL, compress without a dictionary.
* dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
* If this condition is not respected, function will fail (@return an errorCode).
* The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
* but it's not recommended, as it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError()) */
LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const LZ4F_CDict* cdict,
const LZ4F_preferences_t* preferencesPtr);
/*! LZ4F_compressBegin_usingCDict() :
* Inits streaming dictionary compression, and writes the frame header into dstBuffer.
* dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
* `prefsPtr` is optional : you may provide NULL as argument,
* however, it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer for the header,
* or an error code (which can be tested using LZ4F_isError()) */
LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
LZ4F_cctx* cctx,
void* dstBuffer, size_t dstCapacity,
const LZ4F_CDict* cdict,
const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_decompress_usingDict() :
* Same as LZ4F_decompress(), using a predefined dictionary.
* Dictionary is used "in place", without any preprocessing.
* It must remain accessible throughout the entire frame decoding. */
LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
LZ4F_dctx* dctxPtr,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const void* dict, size_t dictSize,
const LZ4F_decompressOptions_t* decompressOptionsPtr);
#if defined (__cplusplus)

577
lib/lz4hc.c
View File

@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -38,54 +38,43 @@
* Tuning Parameter
***************************************/
/*!
* HEAPMODE :
* Select how default compression function will allocate workplace memory,
* in stack (0:fastest), or in heap (1:requires malloc()).
* Since workplace is rather large, heap mode is recommended.
/*! HEAPMODE :
* Select how default compression function will allocate workplace memory,
* in stack (0:fastest), or in heap (1:requires malloc()).
* Since workplace is rather large, heap mode is recommended.
*/
#ifndef LZ4HC_HEAPMODE
# define LZ4HC_HEAPMODE 1
#endif
/* *************************************
* Dependency
***************************************/
/*=== Dependency ===*/
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
/* *************************************
* Local Compiler Options
***************************************/
/*=== Common LZ4 definitions ===*/
#if defined(__GNUC__)
# pragma GCC diagnostic ignored "-Wunused-function"
#endif
#if defined (__clang__)
# pragma clang diagnostic ignored "-Wunused-function"
#endif
/* *************************************
* Common LZ4 definition
***************************************/
#define LZ4_COMMONDEFS_ONLY
#include "lz4.c"
#include "lz4.c" /* LZ4_count, constants, mem */
/* *************************************
* Local Constants
***************************************/
/*=== Constants ===*/
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
/**************************************
* Local Macros
**************************************/
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
/*=== Macros ===*/
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
@ -108,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
{
U16* const chainTable = hc4->chainTable;
U32* const hashTable = hc4->hashTable;
@ -120,7 +109,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
U32 const h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - hashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
DELTANEXTU16(idx) = (U16)delta;
DELTANEXTU16(chainTable, idx) = (U16)delta;
hashTable[h] = idx;
idx++;
}
@ -128,56 +117,73 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
hc4->nextToUpdate = target;
}
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
/** LZ4HC_countBack() :
* @return : negative value, nb of common bytes before ip/match */
LZ4_FORCE_INLINE
int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
const BYTE* const iMin, const BYTE* const mMin)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const BYTE* const dictBase = hc4->dictBase;
const U32 dictLimit = hc4->dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
U32 matchIndex;
int nbAttempts=maxNbAttempts;
size_t ml=0;
/* HC4 match finder */
LZ4HC_Insert(hc4, ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
while ((matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
if (matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (LZ4_read32(match) == LZ4_read32(ip)))
{
size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
} else {
const BYTE* const match = dictBase + matchIndex;
if (LZ4_read32(match) == LZ4_read32(ip)) {
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iLimit))
mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
matchIndex -= DELTANEXTU16(matchIndex);
}
return (int)ml;
int back=0;
while ( (ip+back > iMin)
&& (match+back > mMin)
&& (ip[back-1] == match[back-1]))
back--;
return back;
}
/* LZ4HC_countPattern() :
* pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */
static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
{
const BYTE* const iStart = ip;
reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32;
FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
while (likely(ip < iEnd-(sizeof(pattern)-1))) {
reg_t const diff = LZ4_read_ARCH(ip) ^ pattern;
if (!diff) { ip+=sizeof(pattern); continue; }
ip += LZ4_NbCommonBytes(diff);
return (unsigned)(ip - iStart);
}
if (LZ4_isLittleEndian()) {
reg_t patternByte = pattern;
while ((ip<iEnd) && (*ip == (BYTE)patternByte)) {
ip++; patternByte >>= 8;
}
} else { /* big endian */
U32 bitOffset = (sizeof(pattern)*8) - 8;
while (ip < iEnd) {
BYTE const byte = (BYTE)(pattern >> bitOffset);
if (*ip != byte) break;
ip ++; bitOffset -= 8;
}
}
return (unsigned)(ip - iStart);
}
/* LZ4HC_reverseCountPattern() :
* pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)
* read using natural platform endianess */
static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
{
const BYTE* const iStart = ip;
while (likely(ip >= iLow+4)) {
if (LZ4_read32(ip-4) != pattern) break;
ip -= 4;
}
{ const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */
while (likely(ip>iLow)) {
if (ip[-1] != *bytePtr) break;
ip--; bytePtr--;
} }
return (unsigned)(iStart - ip);
}
typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
LZ4HC_CCtx_internal* hc4,
const BYTE* const ip,
const BYTE* const iLowLimit,
@ -185,98 +191,176 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
int longest,
const BYTE** matchpos,
const BYTE** startpos,
const int maxNbAttempts)
const int maxNbAttempts,
const int patternAnalysis)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const U32 dictLimit = hc4->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
const BYTE* const dictBase = hc4->dictBase;
U32 matchIndex;
int const delta = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
int delta = (int)(ip-iLowLimit);
U32 const pattern = LZ4_read32(ip);
U32 matchIndex;
repeat_state_e repeat = rep_untested;
size_t srcPatternLength = 0;
DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
/* First Match */
LZ4HC_Insert(hc4, ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
matchIndex, lowLimit);
while ((matchIndex>=lowLimit) && (nbAttempts)) {
DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
nbAttempts--;
if (matchIndex >= dictLimit) {
const BYTE* matchPtr = base + matchIndex;
const BYTE* const matchPtr = base + matchIndex;
if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
if (LZ4_read32(matchPtr) == pattern) {
int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
#if 0
/* more generic but unfortunately slower on clang */
int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
#else
int back = 0;
while ((ip+back > iLowLimit)
&& (matchPtr+back > lowPrefixPtr)
&& (ip[back-1] == matchPtr[back-1]))
while ( (ip+back > iLowLimit)
&& (matchPtr+back > lowPrefixPtr)
&& (ip[back-1] == matchPtr[back-1])) {
back--;
}
#endif
mlt -= back;
if (mlt > longest) {
longest = (int)mlt;
longest = mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
}
}
} }
}
} else {
} else { /* matchIndex < dictLimit */
const BYTE* const matchPtr = dictBase + matchIndex;
if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
size_t mlt;
int back=0;
if (LZ4_read32(matchPtr) == pattern) {
int mlt;
int back = 0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
while ( (ip+back > iLowLimit)
&& (matchIndex+back > lowLimit)
&& (ip[back-1] == matchPtr[back-1]))
back--;
mlt -= back;
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
matchIndex -= DELTANEXTU16(matchIndex);
}
if (mlt > longest) {
longest = mlt;
*matchpos = base + matchIndex + back;
*startpos = ip + back;
} } }
{ U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
matchIndex -= nextOffset;
if (patternAnalysis && nextOffset==1) {
/* may be a repeated pattern */
if (repeat == rep_untested) {
if ( ((pattern & 0xFFFF) == (pattern >> 16))
& ((pattern & 0xFF) == (pattern >> 24)) ) {
repeat = rep_confirmed;
srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4;
} else {
repeat = rep_not;
} }
if ( (repeat == rep_confirmed)
&& (matchIndex >= dictLimit) ) { /* same segment only */
const BYTE* const matchPtr = base + matchIndex;
if (LZ4_read32(matchPtr) == pattern) { /* good candidate */
size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
size_t const currentSegmentLength = backLength + forwardPatternLength;
if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */
&& (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */
} else {
matchIndex -= (U32)backLength; /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
}
} } } }
} /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
return longest;
}
LZ4_FORCE_INLINE
int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
const BYTE* const ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts,
const int patternAnalysis)
{
const BYTE* uselessPtr = ip;
/* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
* but this won't be the case here, as we define iLowLimit==ip,
* so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis);
}
typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
#define LZ4HC_DEBUG 0
#if LZ4HC_DEBUG
static unsigned debug = 0;
#endif
FORCE_INLINE int LZ4HC_encodeSequence (
typedef enum {
noLimit = 0,
limitedOutput = 1,
limitedDestSize = 2,
} limitedOutput_directive;
/* LZ4HC_encodeSequence() :
* @return : 0 if ok,
* 1 if buffer issue detected */
LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
const BYTE** ip,
BYTE** op,
const BYTE** anchor,
int matchLength,
const BYTE* const match,
limitedOutput_directive limitedOutputBuffer,
limitedOutput_directive limit,
BYTE* oend)
{
int length;
BYTE* token;
size_t length;
BYTE* const token = (*op)++;
#if LZ4HC_DEBUG
if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
static const BYTE* start = NULL;
static U32 totalCost = 0;
U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start);
U32 const ll = (U32)(*ip - *anchor);
U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
if (start==NULL) start = *anchor; /* only works for single segment */
//g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
pos,
(U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
cost, totalCost);
totalCost += cost;
#endif
/* Encode Literal length */
length = (int)(*ip - *anchor);
token = (*op)++;
if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
else *token = (BYTE)(length<<ML_BITS);
length = (size_t)(*ip - *anchor);
if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length >= RUN_MASK) {
size_t len = length - RUN_MASK;
*token = (RUN_MASK << ML_BITS);
for(; len >= 255 ; len -= 255) *(*op)++ = 255;
*(*op)++ = (BYTE)len;
} else {
*token = (BYTE)(length << ML_BITS);
}
/* Copy Literals */
LZ4_wildCopy(*op, *anchor, (*op) + length);
@ -286,13 +370,14 @@ FORCE_INLINE int LZ4HC_encodeSequence (
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
length = (int)(matchLength-MINMATCH);
if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
if (length>=(int)ML_MASK) {
assert(matchLength >= MINMATCH);
length = (size_t)(matchLength - MINMATCH);
if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
if (length >= ML_MASK) {
*token += ML_MASK;
length -= ML_MASK;
for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; }
if (length > 254) { length-=255; *(*op)++ = 255; }
for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; }
if (length >= 255) { length -= 255; *(*op)++ = 255; }
*(*op)++ = (BYTE)length;
} else {
*token += (BYTE)(length);
@ -305,27 +390,33 @@ FORCE_INLINE int LZ4HC_encodeSequence (
return 0;
}
/* btopt */
#include "lz4opt.h"
static int LZ4HC_compress_generic (
static int LZ4HC_compress_hashChain (
LZ4HC_CCtx_internal* const ctx,
const char* const source,
char* const dest,
int const inputSize,
int* srcSizePtr,
int const maxOutputSize,
int compressionLevel,
unsigned maxNbAttempts,
limitedOutput_directive limit
)
{
const int inputSize = *srcSizePtr;
const int patternAnalysis = (maxNbAttempts > 64); /* levels 8+ */
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
BYTE* optr = (BYTE*) dest;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
BYTE* oend = op + maxOutputSize;
unsigned maxNbAttempts;
int ml, ml2, ml3, ml0;
const BYTE* ref = NULL;
const BYTE* start2 = NULL;
@ -336,17 +427,14 @@ static int LZ4HC_compress_generic (
const BYTE* ref0;
/* init */
if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL;
if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL;
maxNbAttempts = 1 << (compressionLevel-1);
ctx->end += inputSize;
ip++;
*srcSizePtr = 0;
if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* Main Loop */
while (ip < mflimit) {
ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
if (!ml) { ip++; continue; }
ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
if (ml<MINMATCH) { ip++; continue; }
/* saved, in case we would skip too much */
start0 = ip;
@ -355,11 +443,15 @@ static int LZ4HC_compress_generic (
_Search2:
if (ip+ml < mflimit)
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts);
else ml2 = ml;
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
maxNbAttempts, patternAnalysis);
else
ml2 = ml;
if (ml2 == ml) { /* No better match */
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
optr = op;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
continue;
}
@ -380,11 +472,9 @@ _Search2:
}
_Search3:
/*
* Currently we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
/* At this stage, we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1) */
if ((start2 - ip) < OPTIMAL_ML) {
int correction;
int new_ml = ml;
@ -400,16 +490,21 @@ _Search3:
/* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
if (start2 + ml2 < mflimit)
ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
else ml3 = ml2;
ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
maxNbAttempts, patternAnalysis);
else
ml3 = ml2;
if (ml3 == ml2) { /* No better match : 2 sequences to encode */
/* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
optr = op;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
ip = start2;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
optr = op;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow;
continue;
}
@ -427,7 +522,8 @@ _Search3:
}
}
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
optr = op;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
ip = start3;
ref = ref3;
ml = ml3;
@ -463,7 +559,8 @@ _Search3:
ml = (int)(start2 - ip);
}
}
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
optr = op;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
ip = start2;
ref = ref2;
@ -476,34 +573,114 @@ _Search3:
goto _Search3;
}
_last_literals:
/* Encode Last Literals */
{ int lastRun = (int)(iend - anchor);
if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
/* adapt lastRunSize to fill 'dest' */
lastRunSize = (size_t)(oend - op) - 1;
litLength = (lastRunSize + 255 - RUN_MASK) / 255;
lastRunSize -= litLength;
}
ip = anchor + lastRunSize;
if (lastRunSize >= RUN_MASK) {
size_t accumulator = lastRunSize - RUN_MASK;
*op++ = (RUN_MASK << ML_BITS);
for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
*op++ = (BYTE) accumulator;
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
/* End */
*srcSizePtr = (int) (((const char*)ip) - source);
return (int) (((char*)op)-dest);
_dest_overflow:
if (limit == limitedDestSize) {
op = optr; /* restore correct out pointer */
goto _last_literals;
}
return 0;
}
static int LZ4HC_compress_generic (
LZ4HC_CCtx_internal* const ctx,
const char* const src,
char* const dst,
int* const srcSizePtr,
int const dstCapacity,
int cLevel,
limitedOutput_directive limit
)
{
typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
typedef struct {
lz4hc_strat_e strat;
U32 nbSearches;
U32 targetLength;
} cParams_t;
static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = {
{ lz4hc, 2, 16 }, /* 0, unused */
{ lz4hc, 2, 16 }, /* 1, unused */
{ lz4hc, 2, 16 }, /* 2, unused */
{ lz4hc, 4, 16 }, /* 3 */
{ lz4hc, 8, 16 }, /* 4 */
{ lz4hc, 16, 16 }, /* 5 */
{ lz4hc, 32, 16 }, /* 6 */
{ lz4hc, 64, 16 }, /* 7 */
{ lz4hc, 128, 16 }, /* 8 */
{ lz4hc, 256, 16 }, /* 9 */
{ lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/
{ lz4opt, 512,128 }, /*11 */
{ lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */
};
if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */
if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
ctx->end += *srcSizePtr;
if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */
cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
assert(cLevel >= 0);
assert(cLevel <= LZ4HC_CLEVEL_MAX);
{ cParams_t const cParam = clTable[cLevel];
if (cParam.strat == lz4hc)
return LZ4HC_compress_hashChain(ctx,
src, dst, srcSizePtr, dstCapacity,
cParam.nbSearches, limit);
assert(cParam.strat == lz4opt);
return LZ4HC_compress_optimal(ctx,
src, dst, srcSizePtr, dstCapacity,
cParam.nbSearches, cParam.targetLength, limit,
cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */
}
}
int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4HC_init (ctx, (const BYTE*)src);
if (maxDstSize < LZ4_compressBound(srcSize))
return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
if (dstCapacity < LZ4_compressBound(srcSize))
return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
else
return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
}
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
@ -511,13 +688,22 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int
LZ4_streamHC_t state;
LZ4_streamHC_t* const statePtr = &state;
#endif
int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
free(statePtr);
#endif
return cSize;
}
/* LZ4_compress_HC_destSize() :
* only compatible with regular HC parser */
int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
{
LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
LZ4HC_init(ctx, (const BYTE*) source);
return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
}
/**************************************
@ -525,7 +711,11 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int
**************************************/
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
if (!LZ4_streamHCPtr) return 0; /* support free on NULL */
free(LZ4_streamHCPtr);
return 0;
}
/* initialization */
@ -533,19 +723,26 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
LZ4_streamHCPtr->internal_donotuse.base = NULL;
LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
}
void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
if (compressionLevel < 1) compressionLevel = 1;
if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
}
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
if (dictSize > 64 KB) {
dictionary += dictSize - 64 KB;
dictSize = 64 KB;
}
LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
ctxPtr->end = (const BYTE*)dictionary + dictSize;
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
return dictSize;
}
@ -555,6 +752,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{
if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
@ -565,12 +763,13 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
}
static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
const char* source, char* dest,
int inputSize, int maxOutputSize, limitedOutput_directive limit)
const char* src, char* dst,
int* srcSizePtr, int dstCapacity,
limitedOutput_directive limit)
{
LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
/* auto-init if forgotten */
if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source);
if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
@ -580,30 +779,36 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
}
/* Check if blocks follow each other */
if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src);
/* Check overlapping input/dictionary space */
{ const BYTE* sourceEnd = (const BYTE*) source + inputSize;
{ const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;
const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) {
if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);
}
int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity)
{
if (maxOutputSize < LZ4_compressBound(inputSize))
return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
if (dstCapacity < LZ4_compressBound(srcSize))
return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput);
else
return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit);
}
int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize)
{
return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize);
}
/* dictionary saving */
@ -664,16 +869,20 @@ void* LZ4_createHC (char* inputBuffer)
return hc4;
}
int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; }
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
{
return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit);
int LZ4_freeHC (void* LZ4HC_Data) {
if (!LZ4HC_Data) return 0; /* support free on NULL */
FREEMEM(LZ4HC_Data);
return 0;
}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
{
return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit);
}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel)
{
return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput);
}
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)

159
lib/lz4hc.h
View File

@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@ -39,26 +39,27 @@ extern "C" {
#endif
/* --- Dependency --- */
/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */
/* note : lz4hc requires lz4.h/lz4.c for compilation */
#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */
/* --- Useful constants --- */
#define LZ4HC_MIN_CLEVEL 3
#define LZ4HC_DEFAULT_CLEVEL 9
#define LZ4HC_MAX_CLEVEL 16
#define LZ4HC_CLEVEL_MIN 3
#define LZ4HC_CLEVEL_DEFAULT 9
#define LZ4HC_CLEVEL_OPT_MIN 10
#define LZ4HC_CLEVEL_MAX 12
/*-************************************
* Block Compression
**************************************/
/*! LZ4_compress_HC() :
* Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm.
* Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm.
* `dst` must be already allocated.
* Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
* Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
* `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work.
* Values >LZ4HC_MAX_CLEVEL behave the same as 16.
* Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
* Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
* `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work.
* Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.
* @return : the number of bytes written into 'dst'
* or 0 if compression fails.
*/
@ -71,12 +72,12 @@ LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dst
/*! LZ4_compress_HC_extStateHC() :
* Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.
* Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.
* `state` size is provided by LZ4_sizeofStateHC().
* Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
* Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly).
*/
LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
LZ4LIB_API int LZ4_sizeofStateHC(void);
LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*-************************************
@ -86,10 +87,10 @@ LZ4LIB_API int LZ4_sizeofStateHC(void);
typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */
/*! LZ4_createStreamHC() and LZ4_freeStreamHC() :
* These functions create and release memory for LZ4 HC streaming state.
* Newly created states are automatically initialized.
* Existing states can be re-used several times, using LZ4_resetStreamHC().
* These methods are API and ABI stable, they can be used in combination with a DLL.
* These functions create and release memory for LZ4 HC streaming state.
* Newly created states are automatically initialized.
* Existing states can be re-used several times, using LZ4_resetStreamHC().
* These methods are API and ABI stable, they can be used in combination with a DLL.
*/
LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);
LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
@ -112,29 +113,27 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
Then, use LZ4_compress_HC_continue() to compress each successive block.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success.
'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success.
Because in case of failure, the API does not guarantee context recovery, and context will have to be reset.
If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead.
If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
you must save it to a safer memory space, using LZ4_saveDictHC().
If, for any reason, previous data block can't be preserved unmodified in memory for next compression block,
you can save it to a more stable memory space, using LZ4_saveDictHC().
Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
*/
/*-******************************************
* !!!!! STATIC LINKING ONLY !!!!!
*******************************************/
/*-*************************************
/*-**************************************************************
* PRIVATE DEFINITIONS :
* Do not use these definitions.
* They are exposed to allow static allocation of `LZ4_streamHC_t`.
* Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
**************************************/
****************************************************************/
#define LZ4HC_DICTIONARY_LOGSIZE 16
#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1)
#define LZ4HC_HASH_LOG 15
#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG)
#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
@ -146,14 +145,14 @@ typedef struct
{
uint32_t hashTable[LZ4HC_HASHTABLESIZE];
uint16_t chainTable[LZ4HC_MAXD];
const uint8_t* end; /* next block here to continue on current prefix */
const uint8_t* base; /* All index relative to this position */
const uint8_t* dictBase; /* alternate base for extDict */
uint8_t* inputBuffer; /* deprecated */
uint32_t dictLimit; /* below that point, need extDict */
uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */
uint32_t compressionLevel;
const uint8_t* end; /* next block here to continue on current prefix */
const uint8_t* base; /* All index relative to this position */
const uint8_t* dictBase; /* alternate base for extDict */
uint8_t* inputBuffer; /* deprecated */
uint32_t dictLimit; /* below that point, need extDict */
uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */
int compressionLevel;
} LZ4HC_CCtx_internal;
#else
@ -161,7 +160,7 @@ typedef struct
typedef struct
{
unsigned int hashTable[LZ4HC_HASHTABLESIZE];
unsigned short chainTable[LZ4HC_MAXD];
unsigned short chainTable[LZ4HC_MAXD];
const unsigned char* end; /* next block here to continue on current prefix */
const unsigned char* base; /* All index relative to this position */
const unsigned char* dictBase; /* alternate base for extDict */
@ -169,12 +168,12 @@ typedef struct
unsigned int dictLimit; /* below that point, need extDict */
unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */
unsigned int compressionLevel;
int compressionLevel;
} LZ4HC_CCtx_internal;
#endif
#define LZ4_STREAMHCSIZE 262192
#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
union LZ4_streamHC_u {
size_t table[LZ4_STREAMHCSIZE_SIZET];
@ -196,26 +195,25 @@ union LZ4_streamHC_u {
/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */
/* deprecated compression functions */
/* these functions will trigger warning messages in future releases */
LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Deprecated Streaming functions using older model; should no longer be used */
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
#if defined (__cplusplus)
@ -223,3 +221,52 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(
#endif
#endif /* LZ4_HC_H_19834876238432 */
/*-**************************************************
* !!!!! STATIC LINKING ONLY !!!!!
* Following definitions are considered experimental.
* They should not be linked from DLL,
* as there is no guarantee of API stability yet.
* Prototypes will be promoted to "stable" status
* after successfull usage in real-life scenarios.
***************************************************/
#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */
#ifndef LZ4_HC_SLO_098092834
#define LZ4_HC_SLO_098092834
/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental)
* Will try to compress as much data from `src` as possible
* that can fit into `targetDstSize` budget.
* Result is provided in 2 parts :
* @return : the number of bytes written into 'dst'
* or 0 if compression fails.
* `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
*/
int LZ4_compress_HC_destSize(void* LZ4HC_Data,
const char* src, char* dst,
int* srcSizePtr, int targetDstSize,
int compressionLevel);
/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
* Similar as LZ4_compress_HC_continue(),
* but will read a variable nb of bytes from `src`
* to fit into `targetDstSize` budget.
* Result is provided in 2 parts :
* @return : the number of bytes written into 'dst'
* or 0 if compression fails.
* `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`.
*/
int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
const char* src, char* dst,
int* srcSizePtr, int targetDstSize);
/*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
* It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*()
*/
void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
#endif /* LZ4_HC_SLO_098092834 */
#endif /* LZ4_HC_STATIC_LINKING_ONLY */

356
lib/lz4opt.h Normal file
View File

@ -0,0 +1,356 @@
/*
lz4opt.h - Optimal Mode of LZ4
Copyright (C) 2015-2017, Przemyslaw Skibinski <inikep@gmail.com>
Note : this file is intended to be included within lz4hc.c
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/lz4/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#define LZ4_OPT_NUM (1<<12)
typedef struct {
int price;
int off;
int mlen;
int litlen;
} LZ4HC_optimal_t;
/* price in bytes */
LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
{
int price = litlen;
if (litlen >= (int)RUN_MASK)
price += 1 + (litlen-RUN_MASK)/255;
return price;
}
/* requires mlen >= MINMATCH */
LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
{
int price = 1 + 2 ; /* token + 16-bit offset */
price += LZ4HC_literalsPrice(litlen);
if (mlen >= (int)(ML_MASK+MINMATCH))
price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
return price;
}
/*-*************************************
* Match finder
***************************************/
typedef struct {
int off;
int len;
} LZ4HC_match_t;
LZ4_FORCE_INLINE
LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
const BYTE* ip, const BYTE* const iHighLimit,
int minLen, int nbSearches)
{
LZ4HC_match_t match = { 0 , 0 };
const BYTE* matchPtr = NULL;
/* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
* but this won't be the case here, as we define iLowLimit==ip,
* so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
ip, ip, iHighLimit, minLen, &matchPtr, &ip,
nbSearches, 1 /* patternAnalysis */);
if (matchLength <= minLen) return match;
match.len = matchLength;
match.off = (int)(ip-matchPtr);
return match;
}
static int LZ4HC_compress_optimal (
LZ4HC_CCtx_internal* ctx,
const char* const source,
char* dst,
int* srcSizePtr,
int dstCapacity,
int const nbSearches,
size_t sufficient_len,
limitedOutput_directive limit,
int const fullUpdate
)
{
#define TRAILING_LITERALS 3
LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* this uses a bit too much stack memory to my taste ... */
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + *srcSizePtr;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
BYTE* op = (BYTE*) dst;
BYTE* opSaved = (BYTE*) dst;
BYTE* oend = op + dstCapacity;
/* init */
DEBUGLOG(5, "LZ4HC_compress_optimal");
*srcSizePtr = 0;
if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
/* Main Loop */
assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
while (ip < mflimit) {
int const llen = (int)(ip - anchor);
int best_mlen, best_off;
int cur, last_match_pos = 0;
LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
if (firstMatch.len==0) { ip++; continue; }
if ((size_t)firstMatch.len > sufficient_len) {
/* good enough solution : immediate encoding */
int const firstML = firstMatch.len;
const BYTE* const matchPos = ip - firstMatch.off;
opSaved = op;
if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */
goto _dest_overflow;
continue;
}
/* set prices for first positions (literals) */
{ int rPos;
for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
int const cost = LZ4HC_literalsPrice(llen + rPos);
opt[rPos].mlen = 1;
opt[rPos].off = 0;
opt[rPos].litlen = llen + rPos;
opt[rPos].price = cost;
DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
rPos, cost, opt[rPos].litlen);
} }
/* set prices using initial match */
{ int mlen = MINMATCH;
int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */
int const offset = firstMatch.off;
assert(matchML < LZ4_OPT_NUM);
for ( ; mlen <= matchML ; mlen++) {
int const cost = LZ4HC_sequencePrice(llen, mlen);
opt[mlen].mlen = mlen;
opt[mlen].off = offset;
opt[mlen].litlen = llen;
opt[mlen].price = cost;
DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
mlen, cost, mlen);
} }
last_match_pos = firstMatch.len;
{ int addLit;
for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
opt[last_match_pos+addLit].mlen = 1; /* literal */
opt[last_match_pos+addLit].off = 0;
opt[last_match_pos+addLit].litlen = addLit;
opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
} }
/* check further positions */
for (cur = 1; cur < last_match_pos; cur++) {
const BYTE* const curPtr = ip + cur;
LZ4HC_match_t newMatch;
if (curPtr >= mflimit) break;
DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
cur, opt[cur].price, opt[cur+1].price, cur+1);
if (fullUpdate) {
/* not useful to search here if next position has same (or lower) cost */
if ( (opt[cur+1].price <= opt[cur].price)
/* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */
&& (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )
continue;
} else {
/* not useful to search here if next position has same (or lower) cost */
if (opt[cur+1].price <= opt[cur].price) continue;
}
DEBUGLOG(7, "search at rPos:%u", cur);
if (fullUpdate)
newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
else
/* only test matches of minimum length; slightly faster, but misses a few bytes */
newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
if (!newMatch.len) continue;
if ( ((size_t)newMatch.len > sufficient_len)
|| (newMatch.len + cur >= LZ4_OPT_NUM) ) {
/* immediate encoding */
best_mlen = newMatch.len;
best_off = newMatch.off;
last_match_pos = cur + 1;
goto encode;
}
/* before match : set price with literals at beginning */
{ int const baseLitlen = opt[cur].litlen;
int litlen;
for (litlen = 1; litlen < MINMATCH; litlen++) {
int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);
int const pos = cur + litlen;
if (price < opt[pos].price) {
opt[pos].mlen = 1; /* literal */
opt[pos].off = 0;
opt[pos].litlen = baseLitlen+litlen;
opt[pos].price = price;
DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
pos, price, opt[pos].litlen);
} } }
/* set prices using match at position = cur */
{ int const matchML = newMatch.len;
int ml = MINMATCH;
assert(cur + newMatch.len < LZ4_OPT_NUM);
for ( ; ml <= matchML ; ml++) {
int const pos = cur + ml;
int const offset = newMatch.off;
int price;
int ll;
DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
pos, last_match_pos);
if (opt[cur].mlen == 1) {
ll = opt[cur].litlen;
price = ((cur > ll) ? opt[cur - ll].price : 0)
+ LZ4HC_sequencePrice(ll, ml);
} else {
ll = 0;
price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
}
if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
pos, price, ml);
assert(pos < LZ4_OPT_NUM);
if ( (ml == matchML) /* last pos of last match */
&& (last_match_pos < pos) )
last_match_pos = pos;
opt[pos].mlen = ml;
opt[pos].off = offset;
opt[pos].litlen = ll;
opt[pos].price = price;
} } }
/* complete following positions with literals */
{ int addLit;
for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
opt[last_match_pos+addLit].mlen = 1; /* literal */
opt[last_match_pos+addLit].off = 0;
opt[last_match_pos+addLit].litlen = addLit;
opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
} }
} /* for (cur = 1; cur <= last_match_pos; cur++) */
best_mlen = opt[last_match_pos].mlen;
best_off = opt[last_match_pos].off;
cur = last_match_pos - best_mlen;
encode: /* cur, last_match_pos, best_mlen, best_off must be set */
assert(cur < LZ4_OPT_NUM);
assert(last_match_pos >= 1); /* == 1 when only one candidate */
DEBUGLOG(6, "reverse traversal, looking for shortest path")
DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
{ int candidate_pos = cur;
int selected_matchLength = best_mlen;
int selected_offset = best_off;
while (1) { /* from end to beginning */
int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */
int const next_offset = opt[candidate_pos].off;
DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
opt[candidate_pos].mlen = selected_matchLength;
opt[candidate_pos].off = selected_offset;
selected_matchLength = next_matchLength;
selected_offset = next_offset;
if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
assert(next_matchLength > 0); /* can be 1, means literal */
candidate_pos -= next_matchLength;
} }
/* encode all recorded sequences in order */
{ int rPos = 0; /* relative position (to ip) */
while (rPos < last_match_pos) {
int const ml = opt[rPos].mlen;
int const offset = opt[rPos].off;
if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */
rPos += ml;
assert(ml >= MINMATCH);
assert((offset >= 1) && (offset <= MAX_DISTANCE));
opSaved = op;
if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */
goto _dest_overflow;
} }
} /* while (ip < mflimit) */
_last_literals:
/* Encode Last Literals */
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
/* adapt lastRunSize to fill 'dst' */
lastRunSize = (size_t)(oend - op) - 1;
litLength = (lastRunSize + 255 - RUN_MASK) / 255;
lastRunSize -= litLength;
}
ip = anchor + lastRunSize;
if (lastRunSize >= RUN_MASK) {
size_t accumulator = lastRunSize - RUN_MASK;
*op++ = (RUN_MASK << ML_BITS);
for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
*op++ = (BYTE) accumulator;
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
/* End */
*srcSizePtr = (int) (((const char*)ip) - source);
return (int) ((char*)op-dst);
_dest_overflow:
if (limit == limitedDestSize) {
op = opSaved; /* restore correct out pointer */
goto _last_literals;
}
return 0;
}

47
lib/xxhash.c
View File

@ -113,19 +113,24 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
***************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# define FORCE_INLINE static __forceinline
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
#ifndef XXH_FORCE_INLINE
# ifdef _MSC_VER /* Visual Studio */
# define XXH_FORCE_INLINE static __forceinline
# else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define XXH_FORCE_INLINE static inline __attribute__((always_inline))
# else
# define XXH_FORCE_INLINE static inline
# endif
# else
# define XXH_FORCE_INLINE static
# endif /* __STDC_VERSION__ */
# endif /* _MSC_VER */
#endif /* XXH_FORCE_INLINE */
/* *************************************
* Basic Types
@ -218,7 +223,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
@ -226,7 +231,7 @@ FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_a
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
@ -261,7 +266,7 @@ static U32 XXH32_round(U32 seed, U32 input)
return seed;
}
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
@ -376,7 +381,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s
}
FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@ -446,7 +451,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void*
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{
const BYTE * p = (const BYTE*)state->mem32;
const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
@ -578,7 +583,7 @@ static U64 XXH_swap64 (U64 x)
}
#endif
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
@ -586,7 +591,7 @@ FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_a
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
}
FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
@ -621,7 +626,7 @@ static U64 XXH64_mergeRound(U64 acc, U64 val)
return acc;
}
FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@ -745,7 +750,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long
return XXH_OK;
}
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@ -810,7 +815,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void*
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{
const BYTE * p = (const BYTE*)state->mem64;
const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;

2
programs/.gitignore vendored
View File

@ -1,5 +1,7 @@
# local binary (Makefile)
lz4
unlz4
lz4cat
lz4c
lz4c32
datagen

140
programs/Makefile
View File

@ -1,6 +1,8 @@
# ##########################################################################
# LZ4 programs - Makefile
# Copyright (C) Yann Collet 2011-2016
# Copyright (C) Yann Collet 2011-2017
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
# GPL v2 License
#
@ -27,27 +29,41 @@
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# ##########################################################################
DESTDIR ?=
PREFIX ?= /usr/local
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
VOID := /dev/null
# Version numbers
LZ4DIR := ../lib
LIBVER_SRC := $(LZ4DIR)/lz4.h
LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
CFLAGS ?= -O3 # allows custom optimization flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c))
OBJFILES := $(SRCFILES:.c=.o)
CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
LZ4_VERSION=$(LIBVER)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
EXT :=.exe
VOID := nul
else
EXT =
EXT :=
VOID := /dev/null
endif
@ -59,50 +75,96 @@ all: lz4 lz4c
all32: CFLAGS+=-m32
all32: all
lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
lz4: $(OBJFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4-release: CFLAGS := -O3
lz4-release: DEBUGFLAGS=
lz4-release: lz4
lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o
$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
lz4c: lz4
ln -s lz4 lz4c
lz4c32: CFLAGS+=-m32
lz4c32: lz4
@cp lz4$(EXT) lz4c32$(EXT)
lz4c32: CFLAGS += -m32
lz4c32 : $(SRCFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4.1: lz4.1.md $(LIBVER_SRC)
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
man: lz4.1
clean-man:
rm lz4.1
preview-man: clean-man man
man ./lz4.1
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT)
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
@echo Cleaning completed
#------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
install: lz4$(EXT) lz4c$(EXT)
unlz4: lz4
ln -s lz4 unlz4
lz4cat: lz4
ln -s lz4 lz4cat
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
# support both lower and uppercase (BSD), use uppercase in script
prefix ?= /usr/local
PREFIX ?= $(prefix)
exec_prefix ?= $(PREFIX)
bindir ?= $(exec_prefix)/bin
BINDIR ?= $(bindir)
datarootdir ?= $(PREFIX)/share
mandir ?= $(datarootdir)/man
man1dir ?= $(mandir)/man1
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
MANDIR ?= $(PREFIX)/man/man1
else
MANDIR ?= $(man1dir)
endif
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
endif
INSTALL_PROGRAM ?= $(INSTALL) -m 755
INSTALL_DATA ?= $(INSTALL) -m 644
install: lz4
@echo Installing binaries
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT)
@ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat$(EXT)
@ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4$(EXT)
@install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT)
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4
@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c
@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
@echo Installing man pages
@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
@echo lz4 installation completed
uninstall:
@$(RM) $(DESTDIR)$(BINDIR)/lz4cat$(EXT)
@$(RM) $(DESTDIR)$(BINDIR)/unlz4$(EXT)
@$(RM) $(DESTDIR)$(BINDIR)/lz4$(EXT)
@$(RM) $(DESTDIR)$(BINDIR)/lz4c$(EXT)
@$(RM) $(DESTDIR)$(BINDIR)/lz4cat
@$(RM) $(DESTDIR)$(BINDIR)/unlz4
@$(RM) $(DESTDIR)$(BINDIR)/lz4
@$(RM) $(DESTDIR)$(BINDIR)/lz4c
@$(RM) $(DESTDIR)$(MANDIR)/lz4.1
@$(RM) $(DESTDIR)$(MANDIR)/lz4c.1
@$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1

63
programs/bench.c
View File

@ -24,13 +24,22 @@
*/
/*-************************************
* Compiler options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* *************************************
* Includes
***************************************/
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
#include "platform.h" /* Compiler options */
#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <stdio.h> /* fprintf, fopen, ftello */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include "datagen.h" /* RDG_genBuffer */
@ -59,7 +68,7 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
#define COOLPERIOD_SEC 10
#define DECOMP_MULT 2 /* test decompression DECOMP_MULT times longer than compression */
#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */
#define KB *(1 <<10)
#define MB *(1 <<20)
@ -157,7 +166,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
U32 nbBlocks;
UTIL_time_t ticksPerSecond;
struct compressionParameters compP;
int cfunctionId;
@ -167,10 +175,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
UTIL_initTimer(&ticksPerSecond);
/* Init */
if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1;
if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1;
switch (cfunctionId)
{
#ifdef COMPRESSOR0
@ -220,17 +227,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t cSize = 0;
double ratio = 0.;
UTIL_getTime(&coolTime);
coolTime = UTIL_getTime();
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted | !dCompleted) {
while (!cCompleted || !dCompleted) {
UTIL_time_t clockStart;
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
/* overheat protection */
if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
DISPLAYLEVEL(2, "\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
UTIL_getTime(&coolTime);
coolTime = UTIL_getTime();
}
/* Compression */
@ -238,8 +245,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
UTIL_waitForNextTick();
clockStart = UTIL_getTime();
if (!cCompleted) { /* still some time to do compression tests */
U32 nbLoops = 0;
@ -251,8 +258,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[blockNb].cSize = rSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
@ -273,8 +280,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
UTIL_waitForNextTick();
clockStart = UTIL_getTime();
if (!dCompleted) {
U32 nbLoops = 0;
@ -291,8 +298,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[blockNb].resSize = regenSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < DECOMP_MULT*clockLoop);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
} while (UTIL_clockSpanMicro(clockStart) < DECOMP_MULT*clockLoop);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
totalDTime += clockSpan;
dCompleted = totalDTime>(DECOMP_MULT*maxTime);
@ -384,7 +391,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
SET_HIGH_PRIORITY;
SET_REALTIME_PRIORITY;
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
@ -419,7 +426,10 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */
fileSize = bufferSize-pos;
nbFiles=n;
}
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
@ -446,8 +456,13 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
if (benchedSize==0) EXM_THROW(12, "not enough memory");
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
if (benchedSize > LZ4_MAX_INPUT_SIZE) {
benchedSize = LZ4_MAX_INPUT_SIZE;
DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20));
} else {
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
}
srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */
if (!srcBuffer) EXM_THROW(12, "not enough memory");
@ -494,8 +509,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
{
double const compressibility = (double)g_compressibilityDefault / 100;
if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL;
if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL;
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);

38
programs/datagen.c
View File

@ -26,42 +26,13 @@
/**************************************
* Includes
**************************************/
#include "platform.h" /* Compiler options, SET_BINARY_MODE */
#include "util.h" /* U32 */
#include <stdlib.h> /* malloc */
#include <stdio.h> /* FILE, fwrite */
#include <string.h> /* memcpy */
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
#else
# define SET_BINARY_MODE(file)
#endif
/**************************************
* Constants
**************************************/
@ -148,7 +119,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
}
/* init */
if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1;
if (pos==0) {
buffPtr[0] = RDG_genChar(seed, lt);
pos=1;
}
/* Generate compressible data */
while (pos < buffSize)

406
programs/lz4.1
View File

@ -1,251 +1,223 @@
\."
\." lz4.1: This is a manual page for 'lz4' program. This file is part of the
\." lz4 <http://www.lz4.org/> project.
\." Author: Yann Collet
\."
.
\." No hyphenation
.hy 0
.nr HY 0
.TH "LZ4" "1" "2018-01-13" "lz4 1.8.1" "User Commands"
.
.TH lz4 "1" "2015-03-21" "lz4" "User Commands"
.SH NAME
\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files
.SH "NAME"
\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
.
.SH "SYNOPSIS"
\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR
.
.P
\fBunlz4\fR is equivalent to \fBlz4 \-d\fR
.
.P
\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR
.
.P
When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\.
.
.SH "DESCRIPTION"
\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\.
.
.SS "Difference between lz4 and gzip"
\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are :
.
.IP "\(bu" 4
\fBlz4\fR preserves original files
.
.IP "\(bu" 4
\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files)
.
.IP "\(bu" 4
\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2
.
.IP "\(bu" 4
\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
.
.IP "\(bu" 4
\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
.
.IP "\(bu" 4
If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\.
.
.IP "\(bu" 4
If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\.
.
.IP "\(bu" 4
As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
.
.IP "\(bu" 4
Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, you should provide a destination file: \fBnohup lz4 file file\.lz4\fR, because \fBnohup\fR writes the specified command\'s output to a file\.
.
.IP "" 0
.
.P
Default behaviors can be modified by opt\-in commands, detailed below\.
.
.IP "\(bu" 4
\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
.
.IP "\(bu" 4
Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\.
.
.IP "\(bu" 4
It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\.
.
.IP "\(bu" 4
Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\.
.
.IP "" 0
.
.SS "Concatenation of \.lz4 files"
It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example:
.
.IP "" 4
.
.nf
.SH SYNOPSIS
.TP 5
\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
.PP
.B unlz4
is equivalent to
.BR "lz4 \-d"
.br
.B lz4cat
is equivalent to
.BR "lz4 \-dcfm"
.br
.PP
When writing scripts that need to decompress files,
it is recommended to always use the name
.B lz4
with appropriate arguments
.RB ( "lz4 \-d"
or
.BR "lz4 \-dc" )
instead of the names
.B unlz4
and
.BR lz4cat .
.SH DESCRIPTION
.PP
\fBlz4\fR is an extremely fast lossless compression algorithm,
based on \fBbyte-aligned LZ77\fR family of compression scheme.
\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs.
It features an extremely fast decoder, with speed in multiple GB/s per core,
typically reaching RAM speed limit on multi-core systems.
The native file format is the
.B .lz4
format.
.B lz4
supports a command line syntax similar \fIbut not identical\fR to
.BR gzip (1).
Differences are :
\fBlz4\fR preserves original files
\fBlz4\fR compresses a single file by default (use \fB-m\fR for multiple files)
\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2
When no destination name is provided, compressed file name receives a \fB.lz4\fR suffix
When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB-c\fR)
Therefore \fBlz4 file > /dev/null\fR will not create \fBfile.lz4\fR
\fBlz4 file\fR shows real-time statistics during compression (use \fB-q\fR to silent them)
Default behaviors can be modified by opt-in commands, described below.
\fBlz4 --quiet --multiple\fR more closely mimics \fBgzip\fR behavior.
.SS "Concatenation of .lz4 files"
It is possible to concatenate
.B .lz4
files as is.
.B lz4
will decompress such files as if they were a single
.B .lz4
file. For example:
lz4 file1 > foo.lz4
lz4 file2 >> foo.lz4
then
lz4cat foo.lz4
is equivalent to :
cat file1 file2
.PP
.SH OPTIONS
lz4 file1 > foo\.lz4
lz4 file2 >> foo\.lz4
.
.fi
.
.IP "" 0
.
.P
Then \fBlz4cat foo\.lz4\fR is equivalent to \fBcat file1 file2\fR\.
.
.SH "OPTIONS"
.
.SS "Short commands concatenation"
In some cases, some options can be expressed using short command
.B "-x"
or long command
.B "--long-word" .
Short commands can be concatenated together. For example,
.B "-d -c"
is equivalent to
.B "-dc" .
Long commands cannot be concatenated.
They must be clearly separated by a space.
In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\.
.
.SS "Multiple commands"
When multiple contradictory commands are issued on a same command line,
only the latest one will be applied.
When multiple contradictory commands are issued on a same command line, only the latest one will be applied\.
.
.SS "Operation mode"
.
.TP
.BR \-z ", " \-\-compress
Compress.
This is the default operation mode
when no operation mode option is specified ,
no other operation mode is implied from the command name
(for example,
.B unlz4
implies
.B \-\-decompress ),
nor from the input file name
(for example, a file extension
.B .lz4
implies
.B \-\-decompress
by default).
.B -z
can also be used to force compression of an already compressed
.B .lz4
file.
\fB\-z\fR \fB\-\-compress\fR
Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\.
.
.TP
.BR \-d ", " \-\-decompress ", " \-\-uncompress
Decompress.
.B --decompress
is also the default operation when the input filename has an
.B .lz4
extension.
\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR
Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\.
.
.TP
.BR \-t ", " \-\-test
Test the integrity of compressed
.B .lz4
files.
The decompressed data is discarded.
No files are created nor removed.
\fB\-t\fR \fB\-\-test\fR
Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\.
.
.TP
.BR \-b#
Benchmark mode, using # compression level.
\fB\-b#\fR
Benchmark mode, using \fB#\fR compression level\.
.
.SS "Operation modifiers"
.TP
.B \-#
compression level, with # being any value from 1 to 16.
Higher values trade compression speed for compression ratio.
Values above 16 are considered the same as 16.
Recommended values are 1 for fast compression (default), and 9 for high compression.
Speed/compression trade-off will vary depending on data to compress.
Decompression speed remains fast at all settings.
.TP
.BR \-f ", " --[no-]force
This option has several effects:
.RS
.IP \(bu 3
If the target file already exists,
overwrite it without prompting.
.IP \(bu 3
When used with
.B \-\-decompress
and
.B lz4
cannot recognize the type of the source file,
copy the source file as is to standard output.
This allows
.B lz4cat
.B \-\-force
to be used like
.BR cat (1)
for files that have not been compressed with
.BR lz4 .
.RE
.TP
.BR \-c ", " \--stdout ", " \--to-stdout
force write to standard output, even if it is the console
.TP
.BR \-m ", " \--multiple
Multiple file names.
By default, the second filename is used as the destination filename for the compressed file.
With
.B -m
, it is possible to specify any number of input filenames. Each of them will be compressed
independently, and the resulting name of each compressed file will be
.B filename.lz4
.
.TP
.B \-B#
block size [4-7](default : 7)
B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
\fB\-#\fR
Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\.
.
.TP
.B \-BD
block dependency (improves compression ratio on small blocks)
\fB\-f\fR \fB\-\-[no\-]force\fR
This option has several effects:
.
.IP
If the target file already exists, overwrite it without prompting\.
.
.IP
When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\.
.
.TP
.B \--[no-]frame-crc
select frame checksum (default:enabled)
\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR
Force write to standard output, even if it is the console\.
.
.TP
.B \--[no-]content-size
header includes original size (default:not present)
Note : this option can only be activated when the original size can be determined,
hence for a file. It won't work with unknown source size, such as stdin or pipe.
\fB\-m\fR \fB\-\-multiple\fR
Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\.
.
.TP
.B \--[no-]sparse
sparse mode support (default:enabled on file, disabled on stdout)
\fB\-r\fR
operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\.
.
.TP
.B \-l
use Legacy format (typically used for Linux Kernel compression)
note : \fB-l\fR is not compatible with \fB-m\fR (\fB--multiple\fR)
\fB\-B#\fR
Block size [4\-7](default : 7)
.
.br
\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB
.
.TP
\fB\-BD\fR
Block Dependency (improves compression ratio on small blocks)
.
.TP
\fB\-\-[no\-]frame\-crc\fR
Select frame checksum (default:enabled)
.
.TP
\fB\-\-[no\-]content\-size\fR
Header includes original size (default:not present)
.
.br
Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\.
.
.TP
\fB\-\-[no\-]sparse\fR
Sparse mode support (default:enabled on file, disabled on stdout)
.
.TP
\fB\-l\fR
Use Legacy format (typically for Linux Kernel compression)
.
.br
Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR
.
.SS "Other options"
.
.TP
.BR \-v ", " --verbose
verbose mode
\fB\-v\fR \fB\-\-verbose\fR
Verbose mode
.
.TP
.BR \-q ", " --quiet
suppress warnings and real-time statistics; specify twice to suppress errors too
\fB\-q\fR \fB\-\-quiet\fR
Suppress warnings and real\-time statistics; specify twice to suppress errors too
.
.TP
.B \-h/\-H ", " --help
display help/long help and exit
\fB\-h\fR \fB\-H\fR \fB\-\-help\fR
Display help/long help and exit
.
.TP
.BR \-V ", " \--version
display Version number and exit
\fB\-V\fR \fB\-\-version\fR
Display Version number and exit
.
.TP
.BR \-k ", " \--keep
Don't delete source file.
This is default behavior anyway, so this option is just for compatibility with gzip/xz.
\fB\-k\fR \fB\-\-keep\fR
Preserve source files (default behavior)
.
.TP
\fB\-\-rm\fR
Delete source files on successful compression or decompression
.
.TP
\fB\-\-\fR
Treat all subsequent arguments as files
.
.SS "Benchmark mode"
.
.TP
.B \-b#
benchmark file(s), using # compression level
\fB\-b#\fR
Benchmark file(s), using # compression level
.
.TP
.B \-e#
benchmark multiple compression levels, from b# to e# (included)
\fB\-e#\fR
Benchmark multiple compression levels, from b# to e# (included)
.
.TP
.B \-i#
minimum evaluation in seconds [1-9] (default : 3)
.TP
.B \-r
operate recursively on directories
.SH BUGS
Report bugs at: https://github.com/Cyan4973/lz4/issues
.SH AUTHOR
\fB\-i#\fR
Minimum evaluation in seconds [1\-9] (default : 3)
.
.SH "BUGS"
Report bugs at: https://github\.com/lz4/lz4/issues
.
.SH "AUTHOR"
Yann Collet

221
programs/lz4.1.md Normal file
View File

@ -0,0 +1,221 @@
lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files
================================================================
SYNOPSIS
--------
`lz4` [*OPTIONS*] [-|INPUT-FILE] <OUTPUT-FILE>
`unlz4` is equivalent to `lz4 -d`
`lz4cat` is equivalent to `lz4 -dcfm`
When writing scripts that need to decompress files,
it is recommended to always use the name `lz4` with appropriate arguments
(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`.
DESCRIPTION
-----------
`lz4` is an extremely fast lossless compression algorithm,
based on **byte-aligned LZ77** family of compression scheme.
`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with
multi-core CPUs.
It features an extremely fast decoder, with speed in multiple GB/s per core,
typically reaching RAM speed limit on multi-core systems.
The native file format is the `.lz4` format.
### Difference between lz4 and gzip
`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`.
Differences are :
* `lz4` preserves original files
* `lz4` compresses a single file by default (see `-m` for multiple files)
* `lz4 file1 file2` means : compress file1 _into_ file2
* `lz4 file.lz4` will default to decompression (use `-z` to force compression)
* `lz4` shows real-time notification statistics
during compression or decompression of a single file
(use `-q` to silence them)
* If no destination name is provided, result is sent to `stdout`
_except if stdout is the console_.
* If no destination name is provided, __and__ if `stdout` is the console,
`file` is compressed into `file.lz4`.
* As a consequence of previous rules, note the following example :
`lz4 file | consumer` sends compressed data to `consumer` through `stdout`,
hence it does _not_ create `file.lz4`.
* Another consequence of those rules is that to run `lz4` under `nohup`,
you should provide a destination file: `nohup lz4 file file.lz4`,
because `nohup` writes the specified command's output to a file.
Default behaviors can be modified by opt-in commands, detailed below.
* `lz4 -m` makes it possible to provide multiple input filenames,
which will be compressed into files using suffix `.lz4`.
Progress notifications are also disabled by default (use `-v` to enable them).
This mode has a behavior which more closely mimics `gzip` command line,
with the main remaining difference being that source files are preserved by default.
* Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files.
* It's possible to opt-in to erase source files
on successful compression or decompression, using `--rm` command.
* Consequently, `lz4 -m --rm` behaves the same as `gzip`.
### Concatenation of .lz4 files
It is possible to concatenate `.lz4` files as is.
`lz4` will decompress such files as if they were a single `.lz4` file.
For example:
lz4 file1 > foo.lz4
lz4 file2 >> foo.lz4
Then `lz4cat foo.lz4` is equivalent to `cat file1 file2`.
OPTIONS
-------
### Short commands concatenation
In some cases, some options can be expressed using short command `-x`
or long command `--long-word`.
Short commands can be concatenated together.
For example, `-d -c` is equivalent to `-dc`.
Long commands cannot be concatenated.
They must be clearly separated by a space.
### Multiple commands
When multiple contradictory commands are issued on a same command line,
only the latest one will be applied.
### Operation mode
* `-z` `--compress`:
Compress.
This is the default operation mode when no operation mode option is
specified, no other operation mode is implied from the command name
(for example, `unlz4` implies `--decompress`),
nor from the input file name
(for example, a file extension `.lz4` implies `--decompress` by default).
`-z` can also be used to force compression of an already compressed
`.lz4` file.
* `-d` `--decompress` `--uncompress`:
Decompress.
`--decompress` is also the default operation when the input filename has an
`.lz4` extension.
* `-t` `--test`:
Test the integrity of compressed `.lz4` files.
The decompressed data is discarded.
No files are created nor removed.
* `-b#`:
Benchmark mode, using `#` compression level.
### Operation modifiers
* `-#`:
Compression level, with # being any value from 1 to 16.
Higher values trade compression speed for compression ratio.
Values above 16 are considered the same as 16.
Recommended values are 1 for fast compression (default),
and 9 for high compression.
Speed/compression trade-off will vary depending on data to compress.
Decompression speed remains fast at all settings.
* `-f` `--[no-]force`:
This option has several effects:
If the target file already exists, overwrite it without prompting.
When used with `--decompress` and `lz4` cannot recognize the type of
the source file, copy the source file as is to standard output.
This allows `lz4cat --force` to be used like `cat (1)` for files
that have not been compressed with `lz4`.
* `-c` `--stdout` `--to-stdout`:
Force write to standard output, even if it is the console.
* `-m` `--multiple`:
Multiple input files.
Compressed file names will be appended a `.lz4` suffix.
This mode also reduces notification level.
`lz4 -m` has a behavior equivalent to `gzip -k`
(it preserves source files by default).
* `-r` :
operate recursively on directories.
This mode also sets `-m` (multiple input files).
* `-B#`:
Block size \[4-7\](default : 7)<br/>
`-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB
* `-BD`:
Block Dependency (improves compression ratio on small blocks)
* `--[no-]frame-crc`:
Select frame checksum (default:enabled)
* `--[no-]content-size`:
Header includes original size (default:not present)<br/>
Note : this option can only be activated when the original size can be
determined, hence for a file. It won't work with unknown source size,
such as stdin or pipe.
* `--[no-]sparse`:
Sparse mode support (default:enabled on file, disabled on stdout)
* `-l`:
Use Legacy format (typically for Linux Kernel compression)<br/>
Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r`
### Other options
* `-v` `--verbose`:
Verbose mode
* `-q` `--quiet`:
Suppress warnings and real-time statistics;
specify twice to suppress errors too
* `-h` `-H` `--help`:
Display help/long help and exit
* `-V` `--version`:
Display Version number and exit
* `-k` `--keep`:
Preserve source files (default behavior)
* `--rm` :
Delete source files on successful compression or decompression
* `--` :
Treat all subsequent arguments as files
### Benchmark mode
* `-b#`:
Benchmark file(s), using # compression level
* `-e#`:
Benchmark multiple compression levels, from b# to e# (included)
* `-i#`:
Minimum evaluation in seconds \[1-9\] (default : 3)
BUGS
----
Report bugs at: https://github.com/lz4/lz4/issues
AUTHOR
------
Yann Collet

253
programs/lz4cli.c
View File

@ -30,58 +30,32 @@
The license of this compression CLI program is GPLv2.
*/
/**************************************
* Tuning parameters
***************************************/
/* ENABLE_LZ4C_LEGACY_OPTIONS :
Control the availability of -c0, -c1 and -hc legacy arguments
Default : Legacy options are disabled */
/* #define ENABLE_LZ4C_LEGACY_OPTIONS */
/**************************************
* Compiler Options
***************************************/
/* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */
#define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within <stdio.h> on unix */
/****************************
* Includes
*****************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include "platform.h" /* Compiler options, IS_CONSOLE */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include <stdio.h> /* fprintf, getchar */
#include <stdlib.h> /* exit, calloc, free */
#include <string.h> /* strcmp, strlen */
#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */
#include "lz4io.h" /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */
#include "lz4hc.h" /* LZ4HC_DEFAULT_CLEVEL */
#include "lz4hc.h" /* LZ4HC_CLEVEL_MAX */
#include "lz4.h" /* LZ4_VERSION_STRING */
/*-************************************
* OS-specific Includes
**************************************/
#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# define IS_CONSOLE(stdStream) 0
#endif
/*****************************
* Constants
******************************/
#define COMPRESSOR_NAME "LZ4 command line interface"
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION_STRING, AUTHOR
#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR
#define LZ4_EXTENSION ".lz4"
#define LZ4CAT "lz4cat"
#define UNLZ4 "unlz4"
#define LZ4_LEGACY "lz4c"
static int g_lz4c_legacy_commands = 0;
#define KB *(1U<<10)
#define MB *(1U<<20)
@ -116,9 +90,6 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow
/*-************************************
* Version modifiers
***************************************/
#define EXTENDED_ARGUMENTS
#define EXTENDED_HELP
#define EXTENDED_FORMAT
#define DEFAULT_COMPRESSOR LZ4IO_compressFilename
#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */
@ -129,19 +100,21 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
*****************************/
static int usage(const char* exeName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [arg] [input] [output]\n", exeName);
DISPLAY( "Usage : \n");
DISPLAY( " %s [arg] [input] [output] \n", exeName);
DISPLAY( "\n");
DISPLAY( "input : a filename\n");
DISPLAY( "input : a filename \n");
DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);
DISPLAY( "Arguments :\n");
DISPLAY( "Arguments : \n");
DISPLAY( " -1 : Fast compression (default) \n");
DISPLAY( " -9 : High compression \n");
DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION);
DISPLAY( " -z : force compression\n");
DISPLAY( " -z : force compression \n");
DISPLAY( " -D FILE: use dictionary in FILE \n");
DISPLAY( " -f : overwrite output without prompting \n");
DISPLAY( " -k : preserve source files(s) (default) \n");
DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
DISPLAY( " -h/-H : display help/long help and exit\n");
DISPLAY( " -h/-H : display help/long help and exit \n");
return 0;
}
@ -151,36 +124,35 @@ static int usage_advanced(const char* exeName)
usage(exeName);
DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n");
DISPLAY( " -V : display Version number and exit\n");
DISPLAY( " -v : verbose mode\n");
DISPLAY( " -V : display Version number and exit \n");
DISPLAY( " -v : verbose mode \n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
DISPLAY( " -t : test compressed file integrity\n");
DISPLAY( " -m : multiple input files (implies automatic output filenames)\n");
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories (sets also -m)\n");
DISPLAY( " -r : operate recursively on directories (sets also -m) \n");
#endif
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
DISPLAY( " -B# : Block size [4-7] (default : 7)\n");
DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n");
DISPLAY( " -B# : Block size [4-7] (default : 7) \n");
DISPLAY( " -BD : Block dependency (improve compression ratio) \n");
DISPLAY( " -BX : enable block checksum (default:disabled) \n");
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( "Benchmark arguments : \n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n");
DISPLAY( " or predefined block size [4-7] (default: 7)\n");
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
DISPLAY( "Legacy arguments :\n");
DISPLAY( " -c0 : fast compression\n");
DISPLAY( " -c1 : high compression\n");
DISPLAY( " -hc : high compression\n");
DISPLAY( " -y : overwrite output without prompting \n");
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
EXTENDED_HELP;
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n");
DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n");
DISPLAY( " or predefined block size [4-7] (default: 7) \n");
if (g_lz4c_legacy_commands) {
DISPLAY( "Legacy arguments : \n");
DISPLAY( " -c0 : fast compression \n");
DISPLAY( " -c1 : high compression \n");
DISPLAY( " -c2,-hc: very high compression \n");
DISPLAY( " -y : overwrite output without prompting \n");
}
return 0;
}
@ -207,7 +179,7 @@ static int usage_longhelp(const char* exeName)
DISPLAY( "Compression levels : \n");
DISPLAY( "---------------------\n");
DISPLAY( "-0 ... -2 => Fast compression, all identicals\n");
DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL);
DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX);
DISPLAY( "\n");
DISPLAY( "stdin, stdout and the console : \n");
DISPLAY( "--------------------------------\n");
@ -231,17 +203,17 @@ static int usage_longhelp(const char* exeName)
DISPLAY( "-------------------------------------\n");
DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
DISPLAY( " generator | %s | consumer \n", exeName);
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
DISPLAY( "\n");
DISPLAY( "***** Warning *****\n");
DISPLAY( "Legacy arguments take precedence. Therefore : \n");
DISPLAY( "---------------------------------\n");
DISPLAY( " %s -hc filename\n", exeName);
DISPLAY( "means 'compress filename in high compression mode'\n");
DISPLAY( "It is not equivalent to :\n");
DISPLAY( " %s -h -c filename\n", exeName);
DISPLAY( "which would display help text and exit\n");
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
if (g_lz4c_legacy_commands) {
DISPLAY( "\n");
DISPLAY( "***** Warning ***** \n");
DISPLAY( "Legacy arguments take precedence. Therefore : \n");
DISPLAY( "--------------------------------- \n");
DISPLAY( " %s -hc filename \n", exeName);
DISPLAY( "means 'compress filename in high compression mode' \n");
DISPLAY( "It is not equivalent to : \n");
DISPLAY( " %s -h -c filename \n", exeName);
DISPLAY( "which displays help text and exits \n");
}
return 0;
}
@ -259,16 +231,43 @@ static void waitEnter(void)
(void)getchar();
}
static const char* lastNameFromPath(const char* path)
{
const char* name = path;
if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
return name;
}
/*! exeNameMatch() :
@return : a non-zero value if exeName matches test, excluding the extension
*/
static int exeNameMatch(const char* exeName, const char* test)
{
return !strncmp(exeName, test, strlen(test)) &&
(exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
}
/*! readU32FromChar() :
@return : unsigned integer value reach from input in `char` format
@return : unsigned integer value read from input in `char` format
allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : this function can overflow if result > MAX_UINT */
Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
result *= 10;
result += **stringPtr - '0';
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
result <<= 10;
if (**stringPtr=='M') result <<= 10;
(*stringPtr)++ ;
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
@ -283,17 +282,19 @@ int main(int argc, const char** argv)
forceStdout=0,
main_pause=0,
multiple_inputs=0,
all_arguments_are_files=0,
operationResult=0;
operationMode_e mode = om_auto;
const char* input_filename = NULL;
const char* output_filename= NULL;
const char* dictionary_filename = NULL;
char* dynNameSpace = NULL;
const char** inFileNames = (const char**) calloc(argc, sizeof(char*));
unsigned ifnIdx=0;
const char nullOutput[] = NULL_OUTPUT;
const char extension[] = LZ4_EXTENSION;
size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
const char* const exeName = argv[0];
const char* const exeName = lastNameFromPath(argv[0]);
#ifdef UTIL_HAS_CREATEFILELIST
const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
@ -305,18 +306,21 @@ int main(int argc, const char** argv)
DISPLAY("Allocation error : not enough memory \n");
return 1;
}
inFileNames[0] = stdinmark;
LZ4IO_setOverwrite(0);
/* lz4cat predefined behavior */
if (!strcmp(exeName, LZ4CAT)) {
/* predefined behaviors, based on binary/link name */
if (exeNameMatch(exeName, LZ4CAT)) {
mode = om_decompress;
LZ4IO_setOverwrite(1);
LZ4IO_setRemoveSrcFile(0);
forceStdout=1;
output_filename=stdoutmark;
displayLevel=1;
multiple_inputs=1;
}
if (!strcmp(exeName, UNLZ4)) { mode = om_decompress; }
if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; }
if (exeNameMatch(exeName, LZ4_LEGACY)) { g_lz4c_legacy_commands=1; }
/* command switches */
for(i=1; i<argc; i++) {
@ -325,7 +329,7 @@ int main(int argc, const char** argv)
if(!argument) continue; /* Protection if argument empty */
/* Short commands (note : aggregated short commands are allowed) */
if (argument[0]=='-') {
if (!all_arguments_are_files && argument[0]=='-') {
/* '-' means stdin/stdout */
if (argument[1]==0) {
if (!input_filename) input_filename=stdinmark;
@ -335,6 +339,7 @@ int main(int argc, const char** argv)
/* long commands (--long-word) */
if (argument[1]=='-') {
if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; }
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
if ((!strcmp(argument, "--decompress"))
|| (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
@ -361,13 +366,14 @@ int main(int argc, const char** argv)
while (argument[1]!=0) {
argument ++;
#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
/* Legacy arguments (-c0, -c1, -hc, -y, -s) */
if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */
if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */
if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
if (g_lz4c_legacy_commands) {
/* Legacy commands (-c0, -c1, -hc, -y) */
if (!strcmp(argument, "c0")) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */
if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */
if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */
if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
}
if ((*argument>='0') && (*argument<='9')) {
cLevel = readU32FromChar(&argument);
@ -392,6 +398,22 @@ int main(int argc, const char** argv)
/* Compression (default) */
case 'z': mode = om_compress; break;
case 'D':
if (argument[1] == '\0') {
/* path is next arg */
if (i + 1 == argc) {
/* there is no next arg */
badusage(exeName);
}
dictionary_filename = argv[++i];
} else {
/* path follows immediately */
dictionary_filename = argument + 1;
}
/* skip to end of argument so that we jump to parsing next argument */
argument += strlen(argument) - 1;
break;
/* Use Legacy format (ex : Linux kernel compression) */
case 'l': legacy_format = 1; blockSize = 8 MB; break;
@ -459,9 +481,10 @@ int main(int argc, const char** argv)
break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1; /* without break */
/* recursive */
case 'r': recursive=1;
#endif
/* fall-through */
/* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */
case 'm': multiple_inputs=1;
break;
@ -473,16 +496,13 @@ int main(int argc, const char** argv)
iters = readU32FromChar(&argument);
argument--;
BMK_setNotificationLevel(displayLevel);
BMK_SetNbSeconds(iters);
BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */
}
break;
/* Pause at the end (hidden option) */
case 'p': main_pause=1; break;
/* Specific commands for customized versions */
EXTENDED_ARGUMENTS;
/* Unrecognised command */
default : badusage(exeName);
}
@ -508,7 +528,20 @@ int main(int argc, const char** argv)
}
DISPLAYLEVEL(3, WELCOME_MESSAGE);
if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10));
#ifdef _POSIX_C_SOURCE
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
#endif
#ifdef _POSIX_VERSION
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
#endif
#ifdef PLATFORM_POSIX_VERSION
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
#endif
#ifdef _FILE_OFFSET_BITS
DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS);
#endif
if ((mode == om_compress) || (mode == om_bench))
DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10));
if (multiple_inputs) {
input_filename = inFileNames[0];
@ -521,8 +554,7 @@ int main(int argc, const char** argv)
free((void*)inFileNames);
inFileNames = extendedFileList;
ifnIdx = fileNamesNb;
}
}
} }
#endif
}
@ -539,6 +571,14 @@ int main(int argc, const char** argv)
mode = om_decompress; /* defer to decompress */
}
if (dictionary_filename) {
if (!strcmp(dictionary_filename, stdinmark) && IS_CONSOLE(stdin)) {
DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1);
}
LZ4IO_setDictionaryFilename(dictionary_filename);
}
/* compress or decompress */
if (!input_filename) input_filename = stdinmark;
/* Check if input is defined as console; trigger an error in this case */
@ -546,9 +586,12 @@ int main(int argc, const char** argv)
DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1);
}
/* if input==stdin and no output defined, stdout becomes default output */
if (!strcmp(input_filename, stdinmark) && !output_filename)
output_filename = stdoutmark;
/* No output filename ==> try to select one automatically (when possible) */
while (!output_filename) {
while ((!output_filename) && (multiple_inputs==0)) {
if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
size_t const inSize = strlen(input_filename);
@ -584,17 +627,18 @@ int main(int argc, const char** argv)
}
/* Check if output is defined as console; trigger an error in this case */
if (!output_filename) output_filename = "*\\dummy^!//";
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) {
DISPLAYLEVEL(1, "refusing to write to console without -c\n");
exit(1);
}
/* Downgrade notification level in stdout and multiple file mode */
if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((multiple_inputs) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
LZ4IO_setNotificationLevel(displayLevel);
if (ifnIdx == 0) multiple_inputs = 0;
if (mode == om_decompress) {
if (multiple_inputs)
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION);
@ -602,7 +646,7 @@ int main(int argc, const char** argv)
operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename);
} else { /* compression is default action */
if (legacy_format) {
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n");
LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
} else {
if (multiple_inputs)
@ -614,12 +658,13 @@ int main(int argc, const char** argv)
_cleanup:
if (main_pause) waitEnter();
if (dynNameSpace) free(dynNameSpace);
free(dynNameSpace);
#ifdef UTIL_HAS_CREATEFILELIST
if (extendedFileList)
if (extendedFileList) {
UTIL_freeFileList(extendedFileList, fileNamesBuf);
else
inFileNames = NULL;
}
#endif
free((void*)inFileNames);
free((void*)inFileNames);
return operationResult;
}

357
programs/lz4io.c
View File

@ -1,6 +1,6 @@
/*
LZ4io.c - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2016
Copyright (C) Yann Collet 2011-2017
GPL v2 License
@ -30,24 +30,26 @@
- The license of this source file is GPLv2.
*/
/**************************************
* Compiler Options
**************************************/
#define _LARGE_FILES /* Large file support on 32-bits AIX */
#define _FILE_OFFSET_BITS 64 /* off_t width */
#define _LARGEFILE_SOURCE
/*-************************************
* Compiler options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
#endif
/*****************************
* Includes
*****************************/
#include "util.h" /* Compiler options, UTIL_getFileStat */
#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */
#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */
#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <string.h> /* strerror, strcmp, strlen */
#include <time.h> /* clock */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
@ -55,33 +57,7 @@
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
#include "lz4frame.h"
/******************************
* OS-specific Includes
******************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# if !defined(__DJGPP__)
# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <winioctl.h> /* FSCTL_SET_SPARSE */
# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */
# define fseek _fseeki64
# endif
# else
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
# define SET_SPARSE_FILE_MODE(file)
# endif
#else
# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__))
# define fseek fseeko
# endif
# define SET_BINARY_MODE(file)
# define SET_SPARSE_FILE_MODE(file)
#endif
#include "lz4frame_static.h"
/*****************************
@ -107,6 +83,7 @@
#define LEGACY_BLOCKSIZE (8 MB)
#define MIN_STREAM_BUFSIZE (192 KB)
#define LZ4IO_BLOCKSIZEID_DEFAULT 7
#define LZ4_MAX_DICT_SIZE (64 KB)
/**************************************
@ -135,6 +112,8 @@ static int g_streamChecksum = 1;
static int g_blockIndependence = 1;
static int g_sparseFileSupport = 1;
static int g_contentSizeFlag = 0;
static int g_useDictionary = 0;
static const char* g_dictionaryFilename = NULL;
/**************************************
@ -167,6 +146,12 @@ static int g_contentSizeFlag = 0;
/* ****************** Parameters ******************** */
/* ************************************************** */
int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) {
g_dictionaryFilename = dictionaryFilename;
g_useDictionary = dictionaryFilename != NULL;
return g_useDictionary;
}
/* Default setting : overwrite = 1; return : overwrite mode (0/1) */
int LZ4IO_setOverwrite(int yes)
{
@ -198,17 +183,17 @@ int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
return g_blockIndependence;
}
/* Default setting : no checksum */
int LZ4IO_setBlockChecksumMode(int xxhash)
/* Default setting : no block checksum */
int LZ4IO_setBlockChecksumMode(int enable)
{
g_blockChecksum = (xxhash != 0);
g_blockChecksum = (enable != 0);
return g_blockChecksum;
}
/* Default setting : checksum enabled */
int LZ4IO_setStreamChecksumMode(int xxhash)
int LZ4IO_setStreamChecksumMode(int enable)
{
g_streamChecksum = (xxhash != 0);
g_streamChecksum = (enable != 0);
return g_streamChecksum;
}
@ -243,11 +228,13 @@ void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
** ************************************************************************ */
static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; }
static int LZ4IO_isSkippableMagicNumber(unsigned int magic) {
return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0;
}
/** LZ4IO_openSrcFile() :
* condition : `dstFileName` must be non-NULL.
* condition : `srcFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* LZ4IO_openSrcFile(const char* srcFileName)
{
@ -316,7 +303,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName)
/* unoptimized version; solves endianess & alignment issues */
static void LZ4IO_writeLE32 (void* p, unsigned value32)
{
unsigned char* dstPtr = (unsigned char*)p;
unsigned char* const dstPtr = (unsigned char*)p;
dstPtr[0] = (unsigned char)value32;
dstPtr[1] = (unsigned char)(value32 >> 8);
dstPtr[2] = (unsigned char)(value32 >> 16);
@ -342,11 +329,11 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE);
FILE* finput;
FILE* foutput;
clock_t end;
clock_t clockEnd;
/* Init */
clock_t const start = clock();
if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC;
clock_t const clockStart = clock();
compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
finput = LZ4IO_openSrcFile(input_filename);
if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename);
@ -361,7 +348,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Write Archive Header */
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
{ size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); }
if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); }
/* Main Loop */
while (1) {
@ -375,24 +362,27 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Compress Block */
outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
compressedfilesize += outSize+4;
DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ",
(int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
LZ4IO_writeLE32(out_buff, outSize);
{ size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(24, "Write error : cannot write compressed block");
if (sizeCheck!=(size_t)(outSize+4))
EXM_THROW(24, "Write error : cannot write compressed block");
} }
if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
/* Status */
end = clock();
if (end==start) end+=1; /* avoid division by zero (speed) */
clockEnd = clock();
if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */
filesize += !filesize; /* avoid division by zero (ratio) */
DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
filesize, compressedfilesize, (double)compressedfilesize / filesize * 100);
{ double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
{ double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds,
(double)filesize / seconds / 1024 / 1024);
}
/* Close & Free */
@ -415,8 +405,79 @@ typedef struct {
void* dstBuffer;
size_t dstBufferSize;
LZ4F_compressionContext_t ctx;
LZ4F_CDict* cdict;
} cRess_t;
static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) {
size_t readSize;
size_t dictEnd = 0;
size_t dictLen = 0;
size_t dictStart;
size_t circularBufSize = LZ4_MAX_DICT_SIZE;
char* circularBuf;
char* dictBuf;
FILE* dictFile;
if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
circularBuf = (char *) malloc(circularBufSize);
if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory");
dictFile = LZ4IO_openSrcFile(dictFilename);
if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
/* opportunistically seek to the part of the file we care about. If this */
/* fails it's not a problem since we'll just read everything anyways. */
if (strcmp(dictFilename, stdinmark)) {
UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END);
}
do {
readSize = fread(circularBuf + dictEnd, 1, circularBufSize - dictEnd, dictFile);
dictEnd = (dictEnd + readSize) % circularBufSize;
dictLen += readSize;
} while (readSize>0);
if (dictLen > LZ4_MAX_DICT_SIZE) {
dictLen = LZ4_MAX_DICT_SIZE;
}
*dictSize = dictLen;
dictStart = (circularBufSize + dictEnd - dictLen) % circularBufSize;
if (dictStart == 0) {
/* We're in the simple case where the dict starts at the beginning of our circular buffer. */
dictBuf = circularBuf;
circularBuf = NULL;
} else {
/* Otherwise, we will alloc a new buffer and copy our dict into that. */
dictBuf = (char *) malloc(dictLen ? dictLen : 1);
if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart));
}
free(circularBuf);
return dictBuf;
}
static LZ4F_CDict* LZ4IO_createCDict(void) {
size_t dictionarySize;
void* dictionaryBuffer;
LZ4F_CDict* cdict;
if (!g_useDictionary) {
return NULL;
}
dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize);
if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
free(dictionaryBuffer);
return cdict;
}
static cRess_t LZ4IO_createCResources(void)
{
const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
@ -432,6 +493,8 @@ static cRess_t LZ4IO_createCResources(void)
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
ress.cdict = LZ4IO_createCDict();
return ress;
}
@ -439,6 +502,10 @@ static void LZ4IO_freeCResources(cRess_t ress)
{
free(ress.srcBuffer);
free(ress.dstBuffer);
LZ4F_freeCDict(ress.cdict);
ress.cdict = NULL;
{ LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx);
if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
}
@ -475,6 +542,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
prefs.compressionLevel = compressionLevel;
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
if (g_contentSizeFlag) {
U64 const fileSize = UTIL_getFileSize(srcFileName);
@ -491,7 +559,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* single-block file */
if (readSize < blockSize) {
/* Compress in single pass */
size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs);
size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
@ -507,7 +575,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* multiple-blocks file */
{
/* Write Archive Header */
size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs);
size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
{ size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
@ -548,7 +616,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf))
if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && strcmp (dstFileName, nulmark) && UTIL_getFileStat(srcFileName, &statbuf))
UTIL_setFileStat(dstFileName, &statbuf);
}
@ -591,12 +659,15 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
char* dstFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
cRess_t const ress = LZ4IO_createCResources();
cRess_t ress;
if (dstFileName == NULL) return ifntSize; /* not enough memory */
ress = LZ4IO_createCResources();
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); }
if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } }
strcpy(dstFileName, inFileNamesTable[i]);
strcat(dstFileName, suffix);
@ -625,16 +696,16 @@ static unsigned LZ4IO_readLE32 (const void* s)
return value32;
}
#define sizeT sizeof(size_t)
#define maskT (sizeT - 1)
static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
{
const size_t sizeT = sizeof(size_t);
const size_t maskT = sizeT -1 ;
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
const size_t* ptrT = bufferT;
size_t bufferSizeT = bufferSize / sizeT;
const size_t* const bufferTEnd = bufferT + bufferSizeT;
static const size_t segmentSizeT = (32 KB) / sizeT;
const size_t segmentSizeT = (32 KB) / sizeT;
if (!g_sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
@ -644,7 +715,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
/* avoid int overflow */
if (storedSkips > 1 GB) {
int const seekResult = fseek(file, 1 GB, SEEK_CUR);
int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR);
if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@ -660,8 +731,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
storedSkips += (unsigned)(nb0T * sizeT);
if (nb0T != seg0SizeT) { /* not all 0s */
int const seekResult = fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
errno = 0;
{ int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
}
storedSkips = 0;
seg0SizeT -= nb0T;
ptrT += nb0T;
@ -679,7 +752,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd) {
int const seekResult = fseek(file, storedSkips, SEEK_CUR);
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
@ -693,7 +766,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
{
if (storedSkips>0) { /* implies g_sparseFileSupport>0 */
int const seekResult = fseek(file, storedSkips-1, SEEK_CUR);
int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR);
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
{ const char lastZeroByte[1] = { 0 };
size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
@ -702,22 +775,19 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
}
static unsigned g_magicRead = 0;
static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */
static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
{
unsigned long long filesize = 0;
char* in_buff;
char* out_buff;
unsigned long long streamSize = 0;
unsigned storedSkips = 0;
/* Allocate Memory */
in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
char* const in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
/* Main Loop */
while (1) {
int decodeSize;
unsigned int blockSize;
/* Block Size */
@ -736,13 +806,12 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
/* Decode Block */
decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
filesize += decodeSize;
/* Write Block */
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips);
}
{ int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
streamSize += decodeSize;
/* Write Block */
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */
} }
if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
@ -751,7 +820,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
free(in_buff);
free(out_buff);
return filesize;
return streamSize;
}
@ -763,8 +832,21 @@ typedef struct {
size_t dstBufferSize;
FILE* dstFile;
LZ4F_decompressionContext_t dCtx;
void* dictBuffer;
size_t dictBufferSize;
} dRess_t;
static void LZ4IO_loadDDict(dRess_t* ress) {
if (!g_useDictionary) {
ress->dictBuffer = NULL;
ress->dictBufferSize = 0;
return;
}
ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize);
if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
}
static const size_t LZ4IO_dBufferSize = 64 KB;
static dRess_t LZ4IO_createDResources(void)
{
@ -781,6 +863,8 @@ static dRess_t LZ4IO_createDResources(void)
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
LZ4IO_loadDDict(&ress);
ress.dstFile = NULL;
return ress;
}
@ -791,6 +875,7 @@ static void LZ4IO_freeDResources(dRess_t ress)
if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
}
@ -804,7 +889,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
{ size_t inSize = MAGICNUMBER_SIZE;
size_t outSize= 0;
LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL);
nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL);
if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
}
@ -823,7 +908,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
/* Decode Input (at least partially) */
size_t remaining = readSize - pos;
decodedBytes = ress.dstBufferSize;
nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL);
if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
@ -882,7 +967,7 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
while (offset > 0) {
unsigned s = offset;
if (s > stepMax) s = stepMax;
errorNb = fseek(fp, (long) s, SEEK_CUR);
errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR);
if (errorNb != 0) break;
offset -= s;
}
@ -894,22 +979,24 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
{
unsigned char MNstore[MAGICNUMBER_SIZE];
unsigned magicNumber;
static unsigned nbCalls = 0;
static unsigned nbFrames = 0;
/* init */
nbCalls++;
nbFrames++;
/* Check Archive Header */
if (g_magicRead) { /* magic number already read from finput (see legacy frame)*/
magicNumber = g_magicRead;
g_magicRead = 0;
magicNumber = g_magicRead;
g_magicRead = 0;
} else {
size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; } /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE)
EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
}
if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
if (LZ4IO_isSkippableMagicNumber(magicNumber))
magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
switch(magicNumber)
{
@ -920,22 +1007,32 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
return LZ4IO_decodeLegacyStream(finput, foutput);
case LZ4IO_SKIPPABLE0:
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); }
{ unsigned const size = LZ4IO_readLE32(MNstore); /* Little Endian format */
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); }
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4)
EXM_THROW(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(MNstore);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
EXM_THROW(43, "Stream error : cannot skip skippable area");
}
return 0;
EXTENDED_FORMAT; /* macro extension for custom formats */
default:
if (nbCalls == 1) { /* just started */
if (nbFrames == 1) { /* just started */
/* Wrong magic number at the beginning of 1st stream */
if (!g_testMode && g_overwrite) {
nbCalls = 0;
nbFrames = 0;
return LZ4IO_passThrough(finput, foutput, MNstore);
}
EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */
EXM_THROW(44,"Unrecognized header : file cannot be decoded");
}
{ long int const position = ftell(finput); /* only works for files < 2 GB */
DISPLAYLEVEL(2, "Stream followed by undecodable data ");
if (position != -1L)
DISPLAYLEVEL(2, "at position %i ", (int)position);
DISPLAYLEVEL(2, "\n");
}
DISPLAYLEVEL(2, "Stream followed by undecodable data\n");
return ENDOFSTREAM;
}
}
@ -944,24 +1041,26 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename)
{
FILE* const foutput = ress.dstFile;
unsigned long long filesize = 0, decodedSize=0;
FILE* finput;
unsigned long long filesize = 0;
/* Init */
finput = LZ4IO_openSrcFile(input_filename);
FILE* const finput = LZ4IO_openSrcFile(input_filename);
if (finput==NULL) return 1;
/* Loop over multiple streams */
do {
decodedSize = selectDecoder(ress, finput, foutput);
if (decodedSize != ENDOFSTREAM)
filesize += decodedSize;
} while (decodedSize != ENDOFSTREAM);
for ( ; ; ) { /* endless loop, see break condition */
unsigned long long const decodedSize =
selectDecoder(ress, finput, foutput);
if (decodedSize == ENDOFSTREAM) break;
filesize += decodedSize;
}
/* Close */
/* Close input */
fclose(finput);
if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */
if (g_removeSrcFile) { /* --rm */
if (remove(input_filename))
EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
}
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
@ -974,21 +1073,26 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename)
{
FILE* foutput;
/* Init */
foutput = LZ4IO_openDstFile(output_filename);
stat_t statbuf;
int stat_result = 0;
FILE* const foutput = LZ4IO_openDstFile(output_filename);
if (foutput==NULL) return 1; /* failure */
if ( strcmp(input_filename, stdinmark)
&& UTIL_getFileStat(input_filename, &statbuf))
stat_result = 1;
ress.dstFile = foutput;
LZ4IO_decompressSrcFile(ress, input_filename, output_filename);
fclose(foutput);
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
if (strcmp (input_filename, stdinmark) && strcmp (output_filename, stdoutmark) && UTIL_getFileStat(input_filename, &statbuf))
UTIL_setFileStat(output_filename, &statbuf);
if ( stat_result != 0
&& strcmp (output_filename, stdoutmark)
&& strcmp (output_filename, nulmark)) {
UTIL_setFileStat(output_filename, &statbuf);
/* should return value be read ? or is silent fail good enough ? */
}
return 0;
@ -1002,10 +1106,9 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename);
{ clock_t const end = clock();
double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds);
}
clock_t const end = clock();
double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds);
LZ4IO_freeDResources(ress);
return missingFiles;
@ -1022,7 +1125,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
size_t const suffixSize = strlen(suffix);
dRess_t ress = LZ4IO_createDResources();
if (outFileName==NULL) exit(1); /* not enough memory */
if (outFileName==NULL) return ifntSize; /* not enough memory */
ress.dstFile = LZ4IO_openDstFile(stdoutmark);
for (i=0; i<ifntSize; i++) {
@ -1032,7 +1135,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark);
continue;
}
if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); }
if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; }
if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;

2
programs/lz4io.h
View File

@ -64,6 +64,8 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
/* ****************** Parameters ******************** */
/* ************************************************** */
int LZ4IO_setDictionaryFilename(const char* dictionaryFilename);
/* Default setting : overwrite = 1;
return : overwrite mode (0/1) */
int LZ4IO_setOverwrite(int yes);

154
programs/platform.h Normal file
View File

@ -0,0 +1,154 @@
/*
platform.h - compiler and OS detection
Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef PLATFORM_H_MODULE
#define PLATFORM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* **************************************
* Compiler Options
****************************************/
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# endif
#endif
/* **************************************
* Detect 64-bit OS
* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
****************************************/
#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
|| defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
|| (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \
|| defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \
|| defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \
|| (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \
|| defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \
|| (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
# if !defined(__64BIT__)
# define __64BIT__ 1
# endif
#endif
/* *********************************************************
* Turn on Large Files support (>4GB) for 32-bit Linux/Unix
***********************************************************/
#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */
# if !defined(_FILE_OFFSET_BITS)
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
# endif
# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */
# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */
# endif
# if defined(_AIX) || defined(__hpux)
# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */
# endif
#endif
/* ************************************************************
* Detect POSIX version
* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
***************************************************************/
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
# define PLATFORM_POSIX_VERSION 200112L
# else
# if defined(__linux__) || defined(__linux)
# ifndef _POSIX_C_SOURCE
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
# endif
# endif
# include <unistd.h> /* declares _POSIX_VERSION */
# if defined(_POSIX_VERSION) /* POSIX compliant */
# define PLATFORM_POSIX_VERSION _POSIX_VERSION
# else
# define PLATFORM_POSIX_VERSION 0
# endif
# endif
#endif
#if !defined(PLATFORM_POSIX_VERSION)
# define PLATFORM_POSIX_VERSION -1
#endif
/*-*********************************************
* Detect if isatty() and fileno() are available
************************************************/
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(WIN32) || defined(_WIN32)
# include <io.h> /* _isatty */
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <stdio.h> /* FILE */
static __inline int IS_CONSOLE(FILE* stdStream)
{
DWORD dummy;
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
}
#else
# define IS_CONSOLE(stdStream) 0
#endif
/******************************
* OS-specific Includes
******************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# if !defined(__DJGPP__)
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <winioctl.h> /* FSCTL_SET_SPARSE */
# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
# else
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
# define SET_SPARSE_FILE_MODE(file)
# endif
#else
# define SET_BINARY_MODE(file)
# define SET_SPARSE_FILE_MODE(file)
#endif
#if defined (__cplusplus)
}
#endif
#endif /* PLATFORM_H_MODULE */

390
programs/util.h
View File

@ -15,11 +15,8 @@
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : https://github.com/lz4/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#ifndef UTIL_H_MODULE
#define UTIL_H_MODULE
@ -27,107 +24,33 @@
extern "C" {
#endif
/* **************************************
* Compiler Options
****************************************/
#if defined(__INTEL_COMPILER)
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */
#endif
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */
#define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
#endif
#endif
/* Unix Large Files support (>4GB) */
#if !defined(__LP64__) /* No point defining Large file for 64 bit */
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */
# define _LARGEFILE_SOURCE /* fseeko, ftello */
# elif !defined(_LARGEFILE64_SOURCE)
# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */
# endif
#endif
/*-****************************************
* Dependencies
******************************************/
#include <stdlib.h> /* features.h with _POSIX_C_SOURCE, malloc */
#include <stdio.h> /* fprintf */
#include <string.h> /* strerr, strlen, memcpy */
#include <stddef.h> /* ptrdiff_t */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#include "platform.h" /* PLATFORM_POSIX_VERSION */
#include <stdlib.h> /* malloc */
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#if defined(_MSC_VER)
#include <sys/utime.h> /* utime */
#include <io.h> /* _chmod */
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
#else
#include <unistd.h> /* chown, stat */
#include <utime.h> /* utime */
# include <unistd.h> /* chown, stat */
# include <utime.h> /* utime */
#endif
#include <time.h> /* time */
#include <time.h> /* time */
#include <errno.h>
/* *************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
/*-****************************************
* Compiler specifics
******************************************/
#if defined(__GNUC__)
# define UTIL_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define UTIL_STATIC static inline
#elif defined(_MSC_VER)
# define UTIL_STATIC static __inline
#else
# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Sleep functions: Windows - Posix - others
******************************************/
#if defined(_WIN32)
# include <windows.h>
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli)
#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__)))
# include <unistd.h>
# include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS)
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else
# define SET_HIGH_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
# endif
#else
# define SET_HIGH_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */
#endif
/*-**************************************************************
* Basic Types
*****************************************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@ -144,43 +67,189 @@ extern "C" {
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
#endif
#endif
/* ************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
***************************************************************/
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
# define UTIL_fseek _fseeki64
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
# define UTIL_fseek fseeko
#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
# define UTIL_fseek fseeko64
#else
# define UTIL_fseek fseek
#endif
/*-****************************************
* Sleep functions: Windows - Posix - others
******************************************/
#if defined(_WIN32)
# include <windows.h>
# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli)
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
# include <unistd.h>
# include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS)
# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else
# define SET_REALTIME_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
# endif
#else
# define SET_REALTIME_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */
#endif
/* *************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
/*-****************************************
* Compiler specifics
******************************************/
#if defined(__INTEL_COMPILER)
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */
#endif
#if defined(__GNUC__)
# define UTIL_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define UTIL_STATIC static inline
#elif defined(_MSC_VER)
# define UTIL_STATIC static __inline
#else
# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Time functions
******************************************/
#if !defined(_WIN32)
typedef clock_t UTIL_time_t;
UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#else
typedef LARGE_INTEGER UTIL_time_t;
UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
#if defined(_WIN32) /* Windows */
typedef LARGE_INTEGER UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
typedef U64 UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
}
#elif (PLATFORM_POSIX_VERSION >= 200112L)
#include <time.h>
typedef struct timespec UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void)
{
UTIL_time_t now;
if (clock_gettime(CLOCK_MONOTONIC, &now))
fprintf(stderr, "ERROR: Failed to get time\n"); /* we could also exit() */
return now;
}
UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
}
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
typedef clock_t UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
/* returns time span in microseconds */
UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond )
UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart)
{
UTIL_time_t clockEnd;
UTIL_getTime(&clockEnd);
return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
UTIL_STATIC void UTIL_waitForNextTick(void)
{
UTIL_time_t clockStart, clockEnd;
UTIL_getTime(&clockStart);
UTIL_time_t const clockStart = UTIL_getTime();
UTIL_time_t clockEnd;
do {
UTIL_getTime(&clockEnd);
} while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
@ -189,21 +258,27 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
* File functions
******************************************/
#if defined(_MSC_VER)
#define chmod _chmod
typedef struct _stat64 stat_t;
#define chmod _chmod
typedef struct __stat64 stat_t;
#else
typedef struct stat stat_t;
#endif
UTIL_STATIC int UTIL_isRegFile(const char* infilename);
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
struct utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
if (!UTIL_isRegFile(filename))
return -1;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
@ -230,13 +305,39 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
}
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
{
stat_t statbuf;
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
}
UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
{
int r;
stat_t statbuf;
#if defined(_MSC_VER)
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
struct __stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
#elif defined(__MINGW32__) && defined (__MSVCRT__)
struct _stati64 statbuf;
r = _stati64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
@ -256,37 +357,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
}
UTIL_STATIC int UTIL_doesFileExists(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
#endif
return 1;
}
UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
/*
* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
@ -307,7 +377,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
{
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
WIN32_FIND_DATA cFile;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
dirLength = (int)strlen(dirName);
@ -319,7 +389,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
path[dirLength+1] = '*';
path[dirLength+2] = 0;
hFile=FindFirstFile(path, &cFile);
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
fprintf(stderr, "Cannot open directory '%s'\n", dirName);
return 0;
@ -356,16 +426,16 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
}
}
free(path);
} while (FindNextFile(hFile, &cFile));
} while (FindNextFileA(hFile, &cFile));
FindClose(hFile);
return nbFiles;
}
#elif (defined(__APPLE__) && defined(__MACH__)) || \
((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
{
@ -427,7 +497,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
{
(void)bufStart; (void)bufEnd; (void)pos;
fprintf(stderr, "Directory %s ignored (lz4 compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
return 0;
}
@ -443,16 +513,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
{
size_t pos;
unsigned i, nbFiles;
char *bufend, *buf;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
buf = (char*)malloc(LIST_SIZE_INCREASE);
if (!buf) return NULL;
bufend = buf + LIST_SIZE_INCREASE;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t len = strlen(inputNames[i]);
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
@ -474,8 +543,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++)
{
for (i=0, pos=0; i<nbFiles; i++) {
fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1;
}

346
tests/Makefile
View File

@ -1,6 +1,8 @@
# ##########################################################################
# LZ4 programs - Makefile
# Copyright (C) Yann Collet 2011-2016
# Copyright (C) Yann Collet 2011-2017
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
# GPL v2 License
#
@ -28,30 +30,28 @@
# datagen : generates synthetic data samples for tests & benchmarks
# ##########################################################################
DESTDIR ?=
PREFIX ?= /usr/local
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LIBDIR := ../lib
LZ4DIR := ../lib
PRGDIR := ../programs
VOID := /dev/null
TESTDIR := versionsTest
PYTHON ?= python3
CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wdeclaration-after-statement -Wstrict-prototypes \
DEBUGFLAGS = -g -DLZ4_DEBUG=1
CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
EXT =.exe
VOID = nul
else
EXT =
EXT =
VOID = /dev/null
endif
LZ4 := $(PRGDIR)/lz4$(EXT)
@ -64,46 +64,47 @@ NB_LOOPS ?= -i1
default: all
all: fullbench fuzzer frametest datagen fasttest
all: fullbench fuzzer frametest datagen
all32: CFLAGS+=-m32
all32: all
lz4:
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
lz4c:
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
lz4c unlz4 lz4cat: lz4
ln -sf $(LZ4) $(PRGDIR)/$@
lz4c32: # create a 32-bits version for 32/64 interop tests
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
cp $(LZ4) $(LZ4)c32
$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
fullbench : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c
%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench-lib: fullbench.c $(LIBDIR)/xxhash.c
$(MAKE) -C $(LIBDIR) liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4.a
$(LZ4DIR)/liblz4.a:
$(MAKE) -C $(LZ4DIR) liblz4.a
fullbench-dll: fullbench.c $(LIBDIR)/xxhash.c
$(MAKE) -C $(LIBDIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LIBDIR)/dll/liblz4.dll
fuzzer : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o fuzzer.c
fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
frametest: $(LIBDIR)/lz4frame.o $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o frametest.c
fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fasttest: $(LIBDIR)/lz4.o fasttest.c
frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
datagen : $(PRGDIR)/datagen.c datagencli.c
$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
clean:
@$(MAKE) -C $(LIBDIR) $@ > $(VOID)
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
fullbench-dll$(EXT) fullbench-lib$(EXT) \
@ -119,140 +120,216 @@ versionsTest:
$(PYTHON) test-lz4-versions.py
#------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
#-----------------------------------------------------------------------------
# validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
MD5:=md5sum
ifneq (,$(filter $(shell uname), Darwin ))
MD5:=md5 -r
endif
test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer
DIFF:=diff
ifneq (,$(filter $(shell uname),SunOS))
DIFF:=gdiff
endif
DD:=dd
test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer
test32: CFLAGS+=-m32
test32: test
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
./datagen -g5M -P100 > tmpSrc
$(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4
diff -s tmpSrc tmpB4
$(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5
diff -s tmpSrc tmpB5
$(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6
diff -s tmpSrc tmpB6
$(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7
diff -s tmpSrc tmpB7
$(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse
diff -s tmpSrc tmpNoSparse
ls -ls tmp*
./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
ls -ls tmpOdd
@$(RM) tmp*
./datagen -g5M -P100 > tmplsdg5M
$(LZ4) -B4D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB4
$(DIFF) -s tmplsdg5M tmplscB4
$(LZ4) -B5D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB5
$(DIFF) -s tmplsdg5M tmplscB5
$(LZ4) -B6D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB6
$(DIFF) -s tmplsdg5M tmplscB6
$(LZ4) -B7D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB7
$(DIFF) -s tmplsdg5M tmplscB7
$(LZ4) tmplsdg5M | $(LZ4) -dv --no-sparse > tmplsnosparse
$(DIFF) -s tmplsdg5M tmplsnosparse
ls -ls tmpls*
./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
ls -ls tmplsodd
@$(RM) tmpls*
@echo "\n Compatibility with Console :"
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
@echo "\n Compatibility with Append :"
./datagen -P100 -g1M > tmp1M
cat tmp1M tmp1M > tmp2M
$(LZ4) -B5 -v tmp1M tmpC
$(LZ4) -d -v tmpC tmpR
$(LZ4) -d -v tmpC >> tmpR
./datagen -P100 -g1M > tmplsdg1M
cat tmplsdg1M tmplsdg1M > tmpls2M
$(LZ4) -B5 -v tmplsdg1M tmplsc
$(LZ4) -d -v tmplsc tmplsr
$(LZ4) -d -v tmplsc >> tmplsr
ls -ls tmp*
diff tmp2M tmpR
@$(RM) tmp*
$(DIFF) tmpls2M tmplsr
@$(RM) tmpls*
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
./datagen -g15M > tmp
$(LZ4) -v tmp | $(LZ4) -t
$(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2
diff -s tmp tmp2
./datagen -g15M > tmplc1
$(LZ4) -v tmplc1 | $(LZ4) -t
$(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2
$(DIFF) -s tmplc1 tmplc2
# test large size [2-4] GB
@./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp
@ls -ls tmp
@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2
@ls -ls tmp2
diff -s tmp tmp2
@$(RM) tmp*
@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1
@ls -ls tmplc1
@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2
@ls -ls tmplc2
$(DIFF) -s tmplc1 tmplc2
@$(RM) tmplc*
test-lz4-frame-concatenation: lz4 datagen
@echo "\n ---- test frame concatenation ----"
@echo -n > empty.test
@echo hi > nonempty.test
cat nonempty.test empty.test nonempty.test > orig.test
@$(LZ4) -zq empty.test > empty.lz4.test
@$(LZ4) -zq nonempty.test > nonempty.lz4.test
cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test
$(LZ4) -d concat.lz4.test > result.test
sdiff orig.test result.test
@$(RM) *.test
@echo -n > tmp-lfc-empty
@echo hi > tmp-lfc-nonempty
cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
@$(LZ4) -zq tmp-lfc-empty > tmp-lfc-empty.lz4
@$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4
cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
$(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result
sdiff tmp-lfc-src tmp-lfc-result
@$(RM) tmp-lfc-*
@echo frame concatenation test completed
test-lz4-multiple: lz4 datagen
@echo "\n ---- test multiple files ----"
@./datagen -s1 > tmp1 2> $(VOID)
@./datagen -s2 -g100K > tmp2 2> $(VOID)
@./datagen -s3 -g1M > tmp3 2> $(VOID)
$(LZ4) -f -m tmp*
ls -ls tmp*
@$(RM) tmp1 tmp2 tmp3
$(LZ4) -df -m *.lz4
ls -ls tmp*
$(LZ4) -f -m tmp1 notHere tmp2; echo $$?
@$(RM) tmp*
@./datagen -s1 > tmp-tlm1 2> $(VOID)
@./datagen -s2 -g100K > tmp-tlm2 2> $(VOID)
@./datagen -s3 -g1M > tmp-tlm3 2> $(VOID)
$(LZ4) -f -m tmp-tlm*
ls -ls tmp-tlm*
@$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
$(LZ4) -df -m tmp-tlm*.lz4
ls -ls tmp-tlm*
$(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$?
@$(RM) tmp-tlm*
test-lz4-basic: lz4 datagen
test-lz4-basic: lz4 datagen unlz4 lz4cat
@echo "\n ---- test lz4 basic compression/decompression ----"
./datagen -g0 | $(LZ4) -v | $(LZ4) -t
./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t
./datagen -g20KB > tmpSrc
$(LZ4) < tmpSrc | $(LZ4) -d > tmpRes
diff -q tmpSrc tmpRes
$(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes
diff -q tmpSrc tmpRes
./datagen | $(LZ4) | $(LZ4) -t
./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt
./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
@$(RM) tmp*
./datagen -g0 | $(LZ4) -v | $(LZ4) -t
./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t
./datagen -g20KB > tmp-tlb-dg20k
$(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
$(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
./datagen | $(LZ4) | $(LZ4) -t
./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt
./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
@echo "hello world" > tmp-tlb-hw
$(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
test ! -f tmp-tlb-hw # must fail (--rm)
test -f tmp-tlb-hw.lz4
$(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world
test -f tmp-tlb-hw.lz4
$(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw
test -f tmp-tlb-hw
test ! -f tmp-tlb-hw.lz4 # must fail (--rm)
test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4)
$(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode
test -f tmp-tlb-hw
test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat)
$(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4
$(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285)
$(DIFF) -q tmp-tlb-hw tmp-tlb3
$(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode
$(DIFF) -q tmp-tlb-hw tmp-tlb2
cp tmp-tlb-hw ./-d
$(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4
test -f ./-d.lz4
test ! -f ./-d
mv ./-d.lz4 ./-z
$(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4
test ! -f ./-z
$(DIFF) -q tmp-tlb-hw tmp-tlb4
$(LZ4) -f tmp-tlb-hw
cat tmp-tlb-hw >> tmp-tlb-hw.lz4
$(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
@$(RM) tmp-tlb*
test-lz4-dict: lz4 datagen
@echo "\n ---- test lz4 compression/decompression with dictionary ----"
./datagen -g16KB > tmp-dict
./datagen -g32KB > tmp-dict-sample-32k
< tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
./datagen -g128MB > tmp-dict-sample-128m
< tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
touch tmp-dict-sample-0
< tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
< tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k
< tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
@echo "\n ---- test lz4 dictionary loading ----"
./datagen -g128KB > tmp-dict-data-128KB
set -e; \
for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
./datagen -g$$l > tmp-dict-$$l; \
$(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
< tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
< tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
done
@$(RM) tmp-dict*
test-lz4-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----"
./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt
@$(RM) tmp*
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
$(LZ4) -bi1
@echo "\n ---- test mode ----"
./datagen | $(LZ4) -t && false || true
./datagen | $(LZ4) -tf && false || true
! ./datagen | $(LZ4) -t
! ./datagen | $(LZ4) -tf
@echo "\n ---- pass-through mode ----"
./datagen | $(LZ4) -d > $(VOID) && false || true
! ./datagen | $(LZ4) -d > $(VOID)
./datagen | $(LZ4) -df > $(VOID)
@echo "Hello World !" > tmp1
$(LZ4) -dcf tmp1
@echo "from underground..." > tmp2
$(LZ4) -dcfm tmp1 tmp2
@echo "\n ---- test cli ----"
$(LZ4) file-does-not-exist && false || true
$(LZ4) -f file-does-not-exist && false || true
$(LZ4) -fm file1-dne file2-dne && false || true
$(LZ4) -fm file1-dne file2-dne && false || true
@echo "Hello World !" > tmp-tlt1
$(LZ4) -dcf tmp-tlt1
@echo "from underground..." > tmp-tlt2
$(LZ4) -dcfm tmp-tlt1 tmp-tlt2
@echo "\n ---- non-existing source ----"
! $(LZ4) file-does-not-exist
! $(LZ4) -f file-does-not-exist
! $(LZ4) -fm file1-dne file2-dne
@$(RM) tmp-tlt
test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \
test-lz4-hugefile
test-lz4-opt-parser: lz4 datagen
@echo "\n ---- test opt-parser ----"
./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t
./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t
./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t
./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t
./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t
./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t
./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt
./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t
./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t
./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t
test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \
test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \
test-lz4-contentSize test-lz4-hugefile test-lz4-dict
@$(RM) tmp*
test-lz4c: lz4c datagen
@echo "\n ---- test lz4c version ----"
@echo "\n ---- test lz4c variant ----"
./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
test-lz4c32: CFLAGS+=-m32
@ -302,25 +379,22 @@ test-frametest: frametest
test-frametest32: CFLAGS += -m32
test-frametest32: test-frametest
test-fasttest: fasttest
./fasttest
test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f tmp $(VOID)
./datagen -g16KB -s2 > tmp2
./datagen -g16KB -s3 > tmp3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple tmp tmp2 tmp3
./datagen -g16MB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t tmp2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
./datagen -g256MB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq tmp $(VOID)
$(RM) tmp*
./datagen -g16KB > ftmdg16K
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
./datagen -g16KB -s2 > ftmdg16K2
./datagen -g16KB -s3 > ftmdg16K3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
./datagen -g16MB > ftmdg16M
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2
./datagen -g256MB > ftmdg256M
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID)
$(RM) ftm*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256

20
tests/datagencli.c
View File

@ -27,30 +27,12 @@
/**************************************
* Includes
**************************************/
#include "util.h" /* U32 */
#include <stdio.h> /* fprintf, stderr */
#include "datagen.h" /* RDG_generate */
#include "lz4.h" /* LZ4_VERSION_STRING */
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
* Constants
**************************************/

131
tests/fasttest.c
View File

@ -1,131 +0,0 @@
/**************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
# define snprintf sprintf_s
#endif
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
#endif
/**************************************
* Includes
**************************************/
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "lz4.h"
/* Returns non-zero on failure. */
int test_compress(const char *input, int inSize, char *output, int outSize)
{
LZ4_stream_t lz4Stream_body = { 0 };
LZ4_stream_t* lz4Stream = &lz4Stream_body;
int inOffset = 0;
int outOffset = 0;
if (inSize & 3) return -1;
while (inOffset < inSize) {
const int length = inSize >> 2;
if (inSize > 1024) return -2;
if (outSize - (outOffset + 8) < LZ4_compressBound(length)) return -3;
{
const int outBytes = LZ4_compress_fast_continue(
lz4Stream, input + inOffset, output + outOffset + 8, length, outSize-outOffset, 1);
if(outBytes <= 0) return -4;
memcpy(output + outOffset, &length, 4); /* input length */
memcpy(output + outOffset + 4, &outBytes, 4); /* output length */
inOffset += length;
outOffset += outBytes + 8;
}
}
if (outOffset + 8 > outSize) return -5;
memset(output + outOffset, 0, 4);
memset(output + outOffset + 4, 0, 4);
return 0;
}
/* Returns non-zero on failure. Not a safe function. */
int test_decompress(const char *uncompressed, const char *compressed)
{
char outBufferA[1024];
char spacing; /* So prefixEnd != dest */
char outBufferB[1024];
char *output = outBufferA;
char *lastOutput = outBufferB;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
int offset = 0;
int unOffset = 0;
int lastBytes = 0;
(void)spacing;
for(;;) {
int32_t bytes;
int32_t unBytes;
/* Read uncompressed size and compressed size */
memcpy(&unBytes, compressed + offset, 4);
memcpy(&bytes, compressed + offset + 4, 4);
offset += 8;
/* Check if we reached end of stream or error */
if(bytes == 0 && unBytes == 0) return 0;
if(bytes <= 0 || unBytes <= 0 || unBytes > 1024) return 1;
/* Put the last output in the dictionary */
LZ4_setStreamDecode(lz4StreamDecode, lastOutput, lastBytes);
/* Decompress */
bytes = LZ4_decompress_fast_continue(
lz4StreamDecode, compressed + offset, output, unBytes);
if(bytes <= 0) return 2;
/* Check result */
{ int const r = memcmp(uncompressed + unOffset, output, unBytes);
if (r) return 3;
}
{ char* const tmp = output; output = lastOutput; lastOutput = tmp; }
offset += bytes;
unOffset += unBytes;
lastBytes = unBytes;
}
}
int main(int argc, char **argv)
{
char input[] =
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello!"
"Hello Hello Hello Hello Hello Hello Hello Hello";
char output[LZ4_COMPRESSBOUND(4096)];
int r;
(void)argc;
(void)argv;
if ((r = test_compress(input, sizeof(input), output, sizeof(output)))) {
return r;
}
if ((r = test_decompress(input, output))) {
return r;
}
return 0;
}

481
tests/frametest.c
View File

@ -35,35 +35,18 @@
/*-************************************
* Includes
**************************************/
#include "util.h" /* U32 */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* fprintf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
#include <assert.h>
#include "lz4frame_static.h"
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
/*-************************************
* Basic Types
**************************************/
#if !defined(__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/* unoptimized version; solves endianess & alignment issues */
static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
{
@ -85,7 +68,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 256 KB;
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
@ -110,7 +92,7 @@ static clock_t g_clockTime = 0;
*****************************************/
static U32 no_prompt = 0;
static U32 displayLevel = 2;
static U32 pause = 0;
static U32 use_pause = 0;
/*-*******************************************************
@ -182,9 +164,12 @@ static unsigned FUZ_highbit(U32 v32)
/*-*******************************************************
* Tests
*********************************************************/
#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error
#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); }
int basicTests(U32 seed, double compressibility)
{
int testResult = 0;
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
void* const compressedBuffer = malloc(cBuffSize);
@ -194,9 +179,10 @@ int basicTests(U32 seed, double compressibility)
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
int basicTests_error = 0;
LZ4F_preferences_t prefs;
memset(&prefs, 0, sizeof(prefs));
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
goto _output_error;
@ -204,49 +190,56 @@ int basicTests(U32 seed, double compressibility)
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
/* LZ4F_compressBound() : special case : srcSize == 0 */
DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
{ size_t const cBound = LZ4F_compressBound(0, NULL);
if (cBound < 64 KB) goto _output_error;
DISPLAYLEVEL(3, " %u \n", (U32)cBound);
}
/* Special case : null-content frame */
testSize = 0;
DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize);
DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
{ LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error; }
CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
{ size_t avail_in = cSize;
LZ4F_frameInfo_t frame_info;
LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) );
}
DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
{ LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error; }
CHECK( LZ4F_freeDecompressionContext(dCtx) );
dCtx = NULL;
/* test one-pass frame compression */
testSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : ");
{ LZ4F_preferences_t fastCompressPrefs;
memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
fastCompressPrefs.compressionLevel = -3;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs));
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
DISPLAYLEVEL(3, "Decompression test : \n");
{ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedBufferSize = cSize;
BYTE* ip = (BYTE*)compressedBuffer;
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "Single Pass decompression : \n");
{ size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
if (LZ4F_isError(decompressError)) goto _output_error; }
{ U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
DISPLAYLEVEL(3, "Single Pass decompression : ");
CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
{ U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
@ -259,8 +252,7 @@ int basicTests(U32 seed, double compressibility)
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
if (LZ4F_isError(decResult)) goto _output_error;
CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL));
if (decResult != missingBytes) {
DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
goto _output_error;
@ -281,72 +273,93 @@ int basicTests(U32 seed, double compressibility)
{ size_t oSize = 0;
size_t iSize = 0;
LZ4F_frameInfo_t fi;
const BYTE* ip = (BYTE*)compressedBuffer;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) );
//DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "get FrameInfo on null input : ");
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
{ size_t nullSize = 0;
size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
LZ4F_getErrorName(fiError));
goto _output_error;
}
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
}
DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
iSize = 6;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
ip += iSize;
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : ");
{ size_t inputSize = 6;
size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize);
if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
goto _output_error;
}
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
}
DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
iSize = 15 - iSize;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) );
DISPLAYLEVEL(3, " correctly decoded \n");
ip += iSize;
}
DISPLAYLEVEL(3, "Byte after byte : \n");
DISPLAYLEVEL(3, "Decode a buggy input : ");
assert(COMPRESSIBLE_NOISE_LENGTH > 64);
assert(cSize > 48);
memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */
memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */
{ size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
size_t cbSize = cSize;
size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
compressedBuffer, &cbSize,
NULL);
if (!LZ4F_isError(decompressError)) goto _output_error;
DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
}
memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */
DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
LZ4F_resetDecompressionContext(dCtx); /* always successful */
DISPLAYLEVEL(3, "Byte after byte : ");
{ BYTE* const ostart = (BYTE*)decodedBuffer;
BYTE* op = ostart;
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
const BYTE* ip = (const BYTE*) compressedBuffer;
const BYTE* const iend = ip + cSize;
while (ip < iend) {
size_t oSize = oend-op;
size_t iSize = 1;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
{ U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH);
}
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
dCtx = NULL;
}
}
DISPLAYLEVEL(3, "Using 64 KB block : \n");
DISPLAYLEVEL(3, "Using 64 KB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
DISPLAYLEVEL(3, "without checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 256 KB block : \n");
DISPLAYLEVEL(3, "Using 256 KB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max256KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Decompression test : \n");
@ -358,17 +371,13 @@ int basicTests(U32 seed, double compressibility)
const BYTE* ip = (const BYTE*)compressedBuffer;
const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
{ LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(createError)) goto _output_error; }
DISPLAYLEVEL(3, "random segment sizes : \n");
DISPLAYLEVEL(3, "random segment sizes : ");
while (ip < iend) {
unsigned const nbBits = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
{ size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(decompressError)) goto _output_error; }
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@ -378,99 +387,269 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
}
{ LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(freeError)) goto _output_error; }
CHECK( LZ4F_freeDecompressionContext(dCtx) );
dCtx = NULL;
}
DISPLAYLEVEL(3, "without checksum : \n");
DISPLAYLEVEL(3, "without checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 1 MB block : \n");
DISPLAYLEVEL(3, "Using 1 MB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max1MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : \n");
DISPLAYLEVEL(3, "without frame checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 4 MB block : \n");
DISPLAYLEVEL(3, "Using 4 MB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max4MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
{ size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity)
cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
DISPLAYLEVEL(3, "without checksum : \n");
DISPLAYLEVEL(3, "without frame checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
{ size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity)
cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
{ size_t errorCode;
DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Decompress with block checksum : ");
{ size_t iSize = cSize;
size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
LZ4F_decompressionContext_t dctx;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
if (decodedSize != testSize) goto _output_error;
if (iSize != cSize) goto _output_error;
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
if (crcDest != crcSrc) goto _output_error;
}
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
CHECK( LZ4F_freeDecompressionContext(dctx) );
}
/* frame content size tests */
{ size_t cErr;
BYTE* const ostart = (BYTE*)compressedBuffer;
BYTE* op = ostart;
errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "compress without frameSize : \n");
DISPLAYLEVEL(3, "compress without frameSize : ");
memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
op += cErr;
CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
op += cErr;
CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with frameSize : \n");
DISPLAYLEVEL(3, "compress with frameSize : ");
prefs.frameInfo.contentSize = testSize;
op = ostart;
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
op += cErr;
CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
op += cErr;
CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with wrong frameSize : \n");
DISPLAYLEVEL(3, "compress with wrong frameSize : ");
prefs.frameInfo.contentSize = testSize+1;
op = ostart;
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
else
goto _output_error;
CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
op += cErr;
CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
op += cErr;
cErr = LZ4F_compressEnd(cctx, op, testSize, NULL);
if (!LZ4F_isError(cErr)) goto _output_error;
DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr));
errorCode = LZ4F_freeCompressionContext(cctx);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_freeCompressionContext(cctx) );
cctx = NULL;
}
/* dictID tests */
{ size_t cErr;
U32 const dictID = 0x99;
CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "insert a dictID : ");
memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
prefs.frameInfo.dictID = dictID;
CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr);
DISPLAYLEVEL(3, "read a dictID : ");
CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) );
if (prefs.frameInfo.dictID != dictID) goto _output_error;
DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
}
/* Dictionary compression test */
{ size_t const dictSize = 63 KB;
size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
size_t cSizeNoDict, cSizeWithDict;
LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
if (cdict == NULL) goto _output_error;
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
CHECK_V(cSizeNoDict,
LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
CNBuffer, dictSize,
NULL, NULL) );
DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
CHECK_V(cSizeWithDict,
LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
CNBuffer, dictSize,
cdict, NULL) );
DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
(unsigned)dictSize, (unsigned)cSizeWithDict);
if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */
crcOrig = XXH64(CNBuffer, dictSize, 0);
DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
{ LZ4F_dctx* dctx;
size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedSize = cSizeWithDict;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress_usingDict(dctx,
decodedBuffer, &decodedSize,
compressedBuffer, &compressedSize,
CNBuffer, dictSize,
NULL) );
if (compressedSize != cSizeWithDict) goto _output_error;
if (decodedSize != dictSize) goto _output_error;
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
CHECK( LZ4F_freeDecompressionContext(dctx) );
}
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
{ size_t cSizeLevelMax;
LZ4F_preferences_t cParams;
memset(&cParams, 0, sizeof(cParams));
cParams.compressionLevel = -3;
CHECK_V(cSizeLevelMax,
LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
CNBuffer, dictSize,
cdict, &cParams) );
DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
}
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : ");
{ size_t cSizeLevelMax;
LZ4F_preferences_t cParams;
memset(&cParams, 0, sizeof(cParams));
cParams.compressionLevel = LZ4F_compressionLevel_max();
CHECK_V(cSizeLevelMax,
LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
CNBuffer, dictSize,
cdict, &cParams) );
DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
}
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : ");
{ size_t cSizeContiguous;
size_t const inSize = dictSize * 3;
size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
LZ4F_preferences_t cParams;
memset(&cParams, 0, sizeof(cParams));
cParams.frameInfo.blockMode = LZ4F_blockLinked;
cParams.frameInfo.blockSizeID = LZ4F_max64KB;
CHECK_V(cSizeContiguous,
LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
CNBuffer, inSize,
cdict, &cParams) );
DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
(unsigned)inSize, (unsigned)cSizeContiguous);
DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : ");
{ LZ4F_dctx* dctx;
size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedSize = cSizeContiguous;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress_usingDict(dctx,
decodedBuffer, &decodedSize,
compressedBuffer, &compressedSize,
CNBuffer, dictSize,
NULL) );
if (compressedSize != cSizeContiguous) goto _output_error;
if (decodedSize != inSize) goto _output_error;
crcOrig = XXH64(CNBuffer, inSize, 0);
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
CHECK( LZ4F_freeDecompressionContext(dctx) );
}
}
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
{ size_t cSizeIndep;
size_t const inSize = dictSize * 3;
size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
LZ4F_preferences_t cParams;
memset(&cParams, 0, sizeof(cParams));
cParams.frameInfo.blockMode = LZ4F_blockIndependent;
cParams.frameInfo.blockSizeID = LZ4F_max64KB;
CHECK_V(cSizeIndep,
LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
CNBuffer, inSize,
cdict, &cParams) );
DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
(unsigned)inSize, (unsigned)cSizeIndep);
DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : ");
{ LZ4F_dctx* dctx;
size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedSize = cSizeIndep;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress_usingDict(dctx,
decodedBuffer, &decodedSize,
compressedBuffer, &compressedSize,
CNBuffer, dictSize,
NULL) );
if (compressedSize != cSizeIndep) goto _output_error;
if (decodedSize != inSize) goto _output_error;
crcOrig = XXH64(CNBuffer, inSize, 0);
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
CHECK( LZ4F_freeDecompressionContext(dctx) );
}
}
LZ4F_freeCDict(cdict);
}
DISPLAYLEVEL(3, "Skippable frame test : \n");
{ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
@ -479,8 +658,7 @@ int basicTests(U32 seed, double compressibility)
BYTE* ip = (BYTE*)compressedBuffer;
BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
/* generate skippable frame */
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
@ -492,8 +670,7 @@ int basicTests(U32 seed, double compressibility)
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@ -508,12 +685,11 @@ int basicTests(U32 seed, double compressibility)
iend = ip+8;
while (ip < iend) {
unsigned nbBits = FUZ_rand(&randState) % maxBits;
unsigned const nbBits = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@ -529,8 +705,7 @@ int basicTests(U32 seed, double compressibility)
size_t iSize = 10;
size_t oSize = 10;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@ -544,10 +719,10 @@ _end:
free(decodedBuffer);
LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
LZ4F_freeCompressionContext(cctx); cctx = NULL;
return testResult;
return basicTests_error;
_output_error:
testResult = 1;
basicTests_error = 1;
DISPLAY("Error detected ! \n");
goto _end;
}
@ -582,6 +757,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
size_t result;
clock_t const startClock = clock();
clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
# undef CHECK
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
@ -618,10 +794,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11);
if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
DISPLAYUPDATE(2, "\r%5u ", testNb);
@ -723,7 +900,7 @@ _end:
free(compressedBuffer);
free(decodedBuffer);
if (pause) {
if (use_pause) {
DISPLAY("press enter to finish \n");
(void)getchar();
}
@ -795,7 +972,7 @@ int main(int argc, const char** argv)
break;
case 'p': /* pause at the end */
argument++;
pause = 1;
use_pause = 1;
break;
case 'i':

63
tests/fullbench.c
View File

@ -23,20 +23,6 @@
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
* Compiler Options
**************************************/
/* Disable some Visual warning messages */
#define _CRT_SECURE_NO_WARNINGS
#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
/* Unix Large Files support (>4GB) */
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
# define _LARGEFILE_SOURCE
# define _FILE_OFFSET_BITS 64
#elif ! defined(__LP64__) // No point defining Large file for 64 bit
# define _LARGEFILE64_SOURCE
#endif
// S_ISREG & gettimeofday() are not supported by MSVC
#if defined(_MSC_VER) || defined(_WIN32)
@ -47,8 +33,10 @@
/**************************************
* Includes
**************************************/
#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */
#include "util.h" /* U32, UTIL_getFileSize */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <stdio.h> /* fprintf, fopen, ftello */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include <string.h> /* strcmp */
@ -61,34 +49,6 @@
#include "xxhash.h"
/**************************************
* Compiler Options
**************************************/
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
* Constants
**************************************/
@ -194,21 +154,6 @@ static size_t BMK_findMaxMem(U64 requiredMem)
}
static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
/*********************************************************
* Benchmark function
*********************************************************/
@ -398,7 +343,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
/* Memory size adjustments */
inFileSize = BMK_GetFileSize(inFileName);
inFileSize = UTIL_getFileSize(inFileName);
if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */
if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }

226
tests/fuzzer.c
View File

@ -1,6 +1,6 @@
/*
fuzzer.c - Fuzzer test tool for LZ4
Copyright (C) Yann Collet 2012-2016
Copyright (C) Yann Collet 2012-2017
GPL v2 License
@ -27,20 +27,23 @@
* Compiler options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */
#endif
#define LZ4_DISABLE_DEPRECATE_WARNINGS
/*-************************************
* Dependencies
**************************************/
#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */
#include "util.h" /* U32 */
#include <stdlib.h>
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
@ -49,19 +52,7 @@
/*-************************************
* Basic Types
**************************************/
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */
#endif
@ -314,6 +305,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
int const dictSize = MIN(dictSizeRand, blockStart);
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
char* const block = ((char*)CNBuffer) + blockStart;
const char* dict = block - dictSize;
int compressedSize, HCcompressedSize;
@ -353,31 +345,67 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
DISPLAYLEVEL(5, " OK \n");
}
else
} else {
DISPLAYLEVEL(5, " \n");
}
} }
/* Test compression HC destSize */
FUZ_DISPLAYTEST;
{ int srcSize = blockSize;
int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
char const endCheck = FUZ_rand(&randState) & 255;
void* ctx = LZ4_createHC(block);
FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
compressedBuffer[targetSize] = endCheck;
ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
compressionLevel, ret, targetSize, srcSize, blockSize);
LZ4_freeHC(ctx);
FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
compressionLevel, ret, targetSize, srcSize, blockSize);
if (targetSize>0) {
/* check correctness */
U32 const crcBase = XXH32(block, srcSize, 0);
char const canary = FUZ_rand(&randState) & 255;
FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
FUZ_DISPLAYTEST;
compressedSize = ret;
decodedBuffer[srcSize] = canary;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
{ U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
}
DISPLAYLEVEL(5, " OK \n");
} else {
DISPLAYLEVEL(5, " \n");
} }
/* Test compression HC */
FUZ_DISPLAYTEST;
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, 9);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
HCcompressedSize = ret;
/* Test compression HC using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
/* Test compression using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
/* Test compression */
FUZ_DISPLAYTEST;
ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
compressedSize = ret;
/* Decompression tests */
@ -433,8 +461,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
// Test decoding with output size being 10 bytes too short => must fail
FUZ_DISPLAYTEST;
if (blockSize>10)
{
if (blockSize>10) {
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
@ -468,22 +495,22 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* Test compression with output size being exactly what's necessary (should work) */
FUZ_DISPLAYTEST;
ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
/* Test compression with output size being exactly what's necessary and external state (should work) */
FUZ_DISPLAYTEST;
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
/* Test HC compression with output size being exactly what's necessary (should work) */
FUZ_DISPLAYTEST;
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
/* Test HC compression with output size being exactly what's necessary (should work) */
FUZ_DISPLAYTEST;
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
/* Test compression with missing bytes into output buffer => must fail */
FUZ_DISPLAYTEST;
@ -492,8 +519,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
compressedBuffer[compressedSize-missingBytes] = 0;
ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes);
FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes)
FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes)
}
/* Test HC compression with missing bytes into output buffer => must fail */
@ -502,9 +529,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
compressedBuffer[HCcompressedSize-missingBytes] = 0;
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes);
FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes)
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes)
}
@ -518,14 +545,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
LZ4_resetStream(&LZ4_stream);
LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */
blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
}
/* Decompress with dictionary as prefix */
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig) {
int i=0;
@ -533,7 +560,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
@ -547,18 +574,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
/* Decompress with dictionary as external */
FUZ_DISPLAYTEST;
@ -603,32 +630,59 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1);
blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
/* Compress HC continue destSize */
FUZ_DISPLAYTEST;
{ int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
int consumedSize = blockSize;
FUZ_DISPLAYTEST;
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
FUZ_DISPLAYTEST;
decodedBuffer[consumedSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
{ U32 const crcSrc = XXH32(block, consumedSize, 0);
U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0);
if (crcSrc!=crcDst)
FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
}
}
/* ***** End of tests *** */
/* Fill stats */
bytes += blockSize;
@ -666,7 +720,7 @@ _output_error:
#define testCompressedSize (128 KB)
#define ringBufferSize (8 KB)
static void FUZ_unitTests(void)
static void FUZ_unitTests(int compressionLevel)
{
const unsigned testNb = 0;
const unsigned seed = 0;
@ -698,7 +752,7 @@ static void FUZ_unitTests(void)
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState);
result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1);
FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!");
result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
@ -728,7 +782,7 @@ static void FUZ_unitTests(void)
memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
@ -761,7 +815,7 @@ static void FUZ_unitTests(void)
/* simple HC compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
@ -772,7 +826,7 @@ static void FUZ_unitTests(void)
/* simple dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
@ -786,7 +840,7 @@ static void FUZ_unitTests(void)
{ int result1, result2;
int segSize = testCompressedSize / 2;
crcOrig = XXH64(testInput + segSize, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, segSize);
result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1);
@ -803,7 +857,7 @@ static void FUZ_unitTests(void)
/* remote dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
@ -824,7 +878,7 @@ static void FUZ_unitTests(void)
int segSize = (FUZ_rand(&randState) & 8191);
int segNb = 1;
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, dict, dictSize);
XXH64_reset(&crcOrigState, 0);
@ -845,7 +899,6 @@ static void FUZ_unitTests(void)
}
dict = dst;
//dict = testInput + segStart;
dictSize = segSize;
dst += segSize + 1;
@ -870,7 +923,7 @@ static void FUZ_unitTests(void)
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
LZ4_setStreamDecode(&decodeState, NULL, 0);
while (iNext + messageSize < testCompressedSize) {
@ -912,35 +965,34 @@ static void FUZ_unitTests(void)
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_resetStreamHC(&sHC, compressionLevel);
LZ4_setStreamDecode(&decodeState, NULL, 0);
#define BSIZE1 65537
#define BSIZE2 16435
/* first block */
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
while (totalMessageSize < 9 MB) {
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@ -967,7 +1019,7 @@ static void FUZ_unitTests(void)
}
}
printf("All unit tests completed successfully \n");
printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel);
return;
_output_error:
exit(1);
@ -1000,7 +1052,7 @@ int main(int argc, const char** argv)
int nbTests = NB_ATTEMPTS;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int pause = 0;
int use_pause = 0;
const char* programName = argv[0];
U32 duration = 0;
@ -1012,7 +1064,7 @@ int main(int argc, const char** argv)
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-') {
if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; }
if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; }
argument++;
while (*argument!=0) {
@ -1028,7 +1080,7 @@ int main(int argc, const char** argv)
case 'p': /* pause at the end */
argument++;
pause=1;
use_pause=1;
break;
case 'i':
@ -1102,7 +1154,7 @@ int main(int argc, const char** argv)
}
}
printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString());
if (!seedset) {
time_t const t = time(NULL);
@ -1113,12 +1165,12 @@ int main(int argc, const char** argv)
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
if ((seedset==0) && (testNb==0)) FUZ_unitTests();
if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); }
if (nbTests<=0) nbTests=1;
{ int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
if (pause) {
if (use_pause) {
DISPLAY("press enter ... \n");
(void)getchar();
}

8
visual/VS2010/datagen/datagen.vcxproj
View File

@ -39,14 +39,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -91,6 +91,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -107,6 +108,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -124,6 +126,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -144,6 +147,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

16
visual/VS2010/frametest/frametest.vcxproj
View File

@ -39,14 +39,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -66,20 +66,20 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@ -91,6 +91,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -107,6 +108,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -124,6 +126,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -144,6 +147,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

16
visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
View File

@ -39,14 +39,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -66,20 +66,20 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@ -91,6 +91,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -109,6 +110,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -128,6 +130,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -150,6 +153,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

16
visual/VS2010/fullbench/fullbench.vcxproj
View File

@ -39,14 +39,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -66,20 +66,20 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@ -91,6 +91,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -107,6 +108,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -124,6 +126,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -144,6 +147,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

16
visual/VS2010/fuzzer/fuzzer.vcxproj
View File

@ -39,14 +39,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -66,20 +66,20 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@ -91,6 +91,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -107,6 +108,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -124,6 +126,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -144,6 +147,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>

8
visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
View File

@ -40,14 +40,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -96,6 +96,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -111,6 +112,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -127,6 +129,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -146,6 +149,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>

8
visual/VS2010/liblz4/liblz4.vcxproj
View File

@ -39,14 +39,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -95,6 +95,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -110,6 +111,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -126,6 +128,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -145,6 +148,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>

16
visual/VS2010/lz4/lz4.vcxproj
View File

@ -29,24 +29,24 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -91,6 +91,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -108,6 +109,7 @@
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -126,6 +128,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -147,6 +150,7 @@
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>