AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #352 from lz4/resetDCtx

Browse Source 
Reset decompression context
This commit is contained in:
Yann Collet 2017-05-11 14:54:00 -07:00 committed by GitHub
commit 5c97cdfa0d
13 changed files with 321 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
Makefile
View File

@ -52,36 +52,39 @@ EXT =
endif
.PHONY: default all lib lz4 clean test versionsTest examples
.PHONY: default
default: lib lz4-release
default:
@$(MAKE) -C $(LZ4DIR)
@$(MAKE) -C $(PRGDIR)
@cp $(PRGDIR)/lz4$(EXT) .
.PHONY: all
all: allmost manuals
all:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
@$(MAKE) -C $(TESTDIR) $@
@$(MAKE) -C $(EXDIR) $@
.PHONY: allmost
allmost: lib lz4 examples
.PHONY: lib
lib:
@$(MAKE) -C $(LZ4DIR)
lz4:
.PHONY: lz4 lz4-release
lz4 lz4-release: lib
@$(MAKE) -C $(PRGDIR) $@
@cp $(PRGDIR)/lz4$(EXT) .
lz4-release:
@$(MAKE) -C $(PRGDIR)
@cp $(PRGDIR)/lz4$(EXT) .
.PHONY: examples
examples: lib lz4
$(MAKE) -C $(EXDIR) test
.PHONY: manuals
manuals:
@$(MAKE) -C contrib/gen_manual $@
.PHONY: clean
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(EXDIR) $@ > $(VOID)
@$(MAKE) -C examples $@ > $(VOID)
@$(MAKE) -C contrib/gen_manual $@
@$(RM) lz4$(EXT)
@echo Cleaning completed
@ -92,17 +95,35 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX
install:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
uninstall:
install uninstall:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
travis-install:
$(MAKE) -j1 install PREFIX=~/install_test_dir
cmake:
@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
endif
#------------------------------------------------------------------------
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
@ -135,31 +156,10 @@ platformTest: clean
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all
$(MAKE) -C $(TESTDIR) test-platform
.PHONY: versionsTest
versionsTest: clean
$(MAKE) -C $(TESTDIR) $@
examples:
$(MAKE) -C $(LZ4DIR)
$(MAKE) -C $(PRGDIR) lz4
$(MAKE) -C examples test
endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
endif
#------------------------------------------------------------------------
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
cmake:
@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
gpptest: clean
g++ -v
CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
@ -173,13 +173,10 @@ gpptest32: clean
CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
c_standards: clean
$(MAKE) all MOREFLAGS="-std=gnu90 -Werror"
$(MAKE) clean
$(MAKE) all MOREFLAGS="-std=c99 -Werror"
$(MAKE) clean
$(MAKE) all MOREFLAGS="-std=gnu99 -Werror"
$(MAKE) clean
$(MAKE) all MOREFLAGS="-std=c11 -Werror"
$(MAKE) clean
# note : lz4 is not C90 compatible, because it requires long long support
CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost
CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost
CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost
endif

2
contrib/gen_manual/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# build artefact
gen_manual

26
contrib/gen_manual/Makefile
View File

@ -35,7 +35,15 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -W
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
LZ4API = ../../lib/lz4.h
LZ4MANUAL = ../../doc/lz4_manual.html
LZ4FAPI = ../../lib/lz4frame.h
LZ4FMANUAL = ../../doc/lz4frame_manual.html
LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LZ4VER := $(shell echo $(LIBVER_SCRIPT))
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@ -45,14 +53,24 @@ EXT =
endif
.PHONY: default gen_manual
.PHONY: default
default: gen_manual
gen_manual: gen_manual.cpp
$(CXX) $(FLAGS) $^ -o $@$(EXT)
$(CXX) $(FLAGS) $^ -o $@$(EXT)
$(LZ4MANUAL) : gen_manual $(LZ4API)
echo "Update lz4 manual in /doc"
./gen_manual $(LZ4VER) $(LZ4API) $@
$(LZ4FMANUAL) : gen_manual $(LZ4FAPI)
echo "Update lz4frame manual in /doc"
./gen_manual $(LZ4VER) $(LZ4FAPI) $@
.PHONY: manuals
manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL)
.PHONY: clean
clean:
@$(RM) gen_manual$(EXT)
@echo Cleaning completed

18
doc/lz4_manual.html
View File

@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>lz4 1.7.5 Manual</title>
<title>1.8.0 Manual</title>
</head>
<body>
<h1>lz4 1.7.5 Manual</h1>
<h1>1.8.0 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@ -170,21 +170,21 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr);
</p></pre><BR>
<pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
<pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
</b><p> Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
Important : Previous data blocks are assumed to still be present and unmodified !
Important : Previous data blocks are assumed to remain present and unmodified !
'dst' buffer must be already allocated.
If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
After an error, the stream status is invalid, and it can only be reset or freed.
</p></pre><BR>
<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
</b><p> If previously compressed data block is not guaranteed to remain available at its memory location,
</b><p> If previously compressed data block is not guaranteed to remain available at its current memory location,
save it into a safer place (char* safeBuffer).
Note : you don't need to call LZ4_loadDict() afterwards,
dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
@return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
</p></pre><BR>

72
doc/lz4frame_manual.html
View File

@ -1,19 +1,23 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>lz4frame 1.7.5 Manual</title>
<title>1.8.0 Manual</title>
</head>
<body>
<h1>lz4frame 1.7.5 Manual</h1>
<h1>1.8.0 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
<li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Error management</a></li>
<li><a href="#Chapter3">Frame compression types</a></li>
<li><a href="#Chapter4">Simple compression function</a></li>
<li><a href="#Chapter5">Advanced compression functions</a></li>
<li><a href="#Chapter6">Decompression functions</a></li>
<li><a href="#Chapter2">Compiler specifics</a></li>
<li><a href="#Chapter3">Error management</a></li>
<li><a href="#Chapter4">Frame compression types</a></li>
<li><a href="#Chapter5">Simple compression function</a></li>
<li><a href="#Chapter6">Advanced compression functions</a></li>
<li><a href="#Chapter7">Resource Management</a></li>
<li><a href="#Chapter8">Compression</a></li>
<li><a href="#Chapter9">Decompression functions</a></li>
<li><a href="#Chapter10">Streaming decompression functions</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@ -22,13 +26,15 @@
of encoding standard metadata alongside LZ4-compressed blocks.
<BR></pre>
<a name="Chapter2"></a><h2>Error management</h2><pre></pre>
<a name="Chapter2"></a><h2>Compiler specifics</h2><pre></pre>
<a name="Chapter3"></a><h2>Error management</h2><pre></pre>
<pre><b>unsigned LZ4F_isError(LZ4F_errorCode_t code); </b>/**< tells if a `LZ4F_errorCode_t` function result is an error code */<b>
</b></pre><BR>
<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code); </b>/**< return error code string; useful for debugging */<b>
</b></pre><BR>
<a name="Chapter3"></a><h2>Frame compression types</h2><pre></pre>
<a name="Chapter4"></a><h2>Frame compression types</h2><pre></pre>
<pre><b>typedef enum {
LZ4F_default=0,
@ -77,7 +83,7 @@
<pre><b>typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; </b>/* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */<b>
int compressionLevel; </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b>
unsigned autoFlush; </b>/* 1 == always flush (reduce usage of tmp buffer) */<b>
unsigned reserved[4]; </b>/* must be zero for forward compatibility */<b>
} LZ4F_preferences_t;
@ -86,7 +92,7 @@
All reserved fields must be set to zero.
</p></pre><BR>
<a name="Chapter4"></a><h2>Simple compression function</h2><pre></pre>
<a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
<pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
@ -105,17 +111,19 @@
</p></pre><BR>
<a name="Chapter5"></a><h2>Advanced compression functions</h2><pre></pre>
<a name="Chapter6"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>typedef struct {
unsigned stableSrc; </b>/* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */<b>
unsigned reserved[3];
} LZ4F_compressOptions_t;
</b></pre><BR>
<a name="Chapter7"></a><h2>Resource Management</h2><pre></pre>
<pre><b>LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
</b><p> The first thing to do is to create a compressionContext object, which will be used in all compression operations.
This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
The function will provide a pointer to a fully allocated LZ4F_cctx object.
If @return != zero, there was an error during context creation.
@ -123,6 +131,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
</p></pre><BR>
<a name="Chapter8"></a><h2>Compression</h2><pre></pre>
<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
</b><p> will write the frame header into dstBuffer.
dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
@ -173,7 +183,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
</p></pre><BR>
<a name="Chapter6"></a><h2>Decompression functions</h2><pre></pre>
<a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre>
<pre><b>typedef struct {
unsigned stableDst; </b>/* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */<b>
@ -181,7 +191,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
} LZ4F_decompressOptions_t;
</b></pre><BR>
<pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
</b><p> Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
@ -192,19 +202,27 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
</p></pre><BR>
<a name="Chapter10"></a><h2>Streaming decompression functions</h2><pre></pre>
<pre><b>size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
</b><p> This function decodes frame header information (such as max blockSize, frame checksum, etc.).
Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
</b><p> This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
Its usage is optional. Extracted information can be useful for allocation purposes, typically.
This function works in 2 situations :
- At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
Input size must be large enough to successfully decode the entire frame header.
Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
It's allowed to provide more input data than this minimum.
- After decoding has been started.
In which case, no input is read, frame parameters are extracted from dctx.
If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
Decompression must resume from this point (srcBuffer + *srcSizePtr).
Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
Decompression must resume from (srcBuffer + *srcSizePtr).
@return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
or an error code which can be tested using LZ4F_isError()
(typically, when there is not enough src bytes to fully decode the frame header)
note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
</p></pre><BR>
@ -227,14 +245,22 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
@return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
Schematically, it's the size of the current (or remaining) compressed block + header of next block.
Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
This is just a hint though, it's always possible to provide any srcSize.
When a frame is fully decoded, @return will be 0 (no more data expected).
If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
After a frame is fully decoded, dctx can be used again to decompress another frame.
After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
</p></pre><BR>
<pre><b>void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); </b>/* always successful */<b>
</b><p> In case of an error, the context is left in "undefined" state.
In which case, it's necessary to reset it, before re-using it.
This method can also be used to abruptly stop an unfinished decompression,
and start a new with the same context.
</p></pre><BR>
</html>
</body>

13
lib/Makefile
View File

@ -46,10 +46,10 @@ BUILD_STATIC:= yes
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@ -70,12 +70,13 @@ endif
LIBLZ4 = liblz4.$(SHARED_EXT_VER)
.PHONY: default
default: lib-release
lib-release: liblz4.a liblz4
lib-release: DEBUGFLAGS :=
lib-release: lib
lib: CFLAGS += $(DEBUGFLAGS)
lib: lib-release
lib: liblz4.a liblz4
all: lib
@ -83,7 +84,7 @@ all32: CFLAGS+=-m32
all32: all
liblz4.a: *.c
ifeq ($(BUILD_STATIC),yes)
ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
@$(AR) rcs $@ *.o

4
lib/lz4.h
View File

@ -88,8 +88,8 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)

200
lib/lz4frame.c
View File

@ -209,7 +209,8 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
{
LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
/* A compilation error here means sizeof(ptrdiff_t) is not large enough */
LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
}
@ -241,7 +242,8 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length)
/*-************************************
* Simple-pass compression functions
**************************************/
static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize)
static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
const size_t srcSize)
{
LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
size_t maxBlockSize = 64 KB;
@ -256,11 +258,13 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
/* LZ4F_compressBound() :
* Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
* prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
* Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
* prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
* Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
* When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered)
static size_t LZ4F_compressBound_internal(size_t srcSize,
const LZ4F_preferences_t* preferencesPtr,
size_t alreadyBuffered)
{
LZ4F_preferences_t prefsNull;
memset(&prefsNull, 0, sizeof(prefsNull));
@ -298,14 +302,14 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
/*! LZ4F_compressFrame() :
* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default.
* The result of the function is the number of bytes written into dstBuffer.
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
*/
* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
* Get the minimum value of dstCapacity by using LZ4F_compressFrameBound().
* The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default.
* The result of the function is the number of bytes written into dstBuffer.
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_cctx_t cctxI;
@ -404,10 +408,10 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
/*! LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
* @return : number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
* @return : number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr)
{
@ -650,13 +654,13 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
/*! LZ4F_flush() :
* Should you need to create compressed data immediately, without waiting for a block to be filled,
* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
* The result of the function is the number of bytes written into dstBuffer
* (it can be zero, this means there was no data left within compressionContext)
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
*/
* Should you need to create compressed data immediately, without waiting for a block to be filled,
* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
* The result of the function is the number of bytes written into dstBuffer
* (it can be zero, this means there was no data left within compressionContext)
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
*/
size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
@ -687,14 +691,14 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
/*! LZ4F_compressEnd() :
* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
* It will flush whatever data remained within compressionContext (like LZ4_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
*/
* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
* It will flush whatever data remained within compressionContext (like LZ4_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
*/
size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
@ -751,11 +755,11 @@ struct LZ4F_dctx_s {
/*! LZ4F_createDecompressionContext() :
* Create a decompressionContext object, which will track all decompression operations.
* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
* Object can later be released using LZ4F_freeDecompressionContext().
* @return : if != 0, there was an error during context creation.
*/
* Create a decompressionContext object, which will track all decompression operations.
* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
* Object can later be released using LZ4F_freeDecompressionContext().
* @return : if != 0, there was an error during context creation.
*/
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
@ -794,17 +798,16 @@ typedef enum {
dstage_skipSkippable
} dStage_t;
LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
{
dctx->dStage = dstage_getHeader;
return 0;
}
/*! LZ4F_headerSize() :
* @return : size of frame header
* or an error code, which can be tested using LZ4F_isError()
*/
* @return : size of frame header
* or an error code, which can be tested using LZ4F_isError()
*/
static size_t LZ4F_headerSize(const void* src, size_t srcSize)
{
/* minimal srcSize to determine header size */
@ -825,13 +828,13 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
/*! LZ4F_decodeHeader() :
input : `src` points at the **beginning of the frame**
output : set internal values of dctx, such as
dctxPtr->frameInfo and dctxPtr->dStage.
Also allocates internal buffers.
@return : nb Bytes read from src (necessarily <= srcSize)
or an error code (testable with LZ4F_isError())
*/
* input : `src` points at the **beginning of the frame**
* output : set internal values of dctx, such as
* dctxPtr->frameInfo and dctxPtr->dStage.
* Also allocates internal buffers.
* @return : nb Bytes read from src (necessarily <= srcSize)
* or an error code (testable with LZ4F_isError())
*/
static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
{
unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID;
@ -913,8 +916,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
* Its usage is optional. The objective is to provide relevant information for allocation purposes.
* This function extracts frame parameters (max blockSize, frame checksum, etc.).
* Usage is optional. Objective is to provide relevant information for allocation purposes.
* This function works in 2 situations :
* - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
* Amount of input data provided must be large enough to successfully decode the frame header.
@ -935,7 +938,8 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
size_t o=0, i=0;
*srcSizePtr = 0;
*frameInfoPtr = dctxPtr->frameInfo;
return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */
/* returns : recommended nb of bytes for LZ4F_decompress() */
return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);
} else {
if (dctxPtr->dStage == dstage_storeHeader) {
/* frame decoding already started, in the middle of header => automatic fail */
@ -945,7 +949,10 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
size_t decodeResult;
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); }
if (*srcSizePtr < hSize) {
*srcSizePtr=0;
return err0r(LZ4F_ERROR_frameHeader_incomplete);
}
decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize);
if (LZ4F_isError(decodeResult)) {
@ -961,10 +968,15 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
/* trivial redirector, for common prototype */
static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
static int LZ4F_decompress_safe (const char* src,
char* dst,
int compressedSize,
int dstCapacity,
const char* dictStart,
int dictSize)
{
(void)dictStart; (void)dictSize;
return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity);
}
@ -1028,22 +1040,22 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate data compressed within srcBuffer.
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
* Remaining data will have to be presented again in a subsequent invocation.
*
* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
* Basically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
* Note that this is just a hint, it's always possible to any srcSize value.
* When a frame is fully decoded, @return will be 0.
* If decompression failed, @return is an error code which can be tested using LZ4F_isError().
*/
* Call this function repetitively to regenerate data compressed within srcBuffer.
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
* Remaining data will have to be presented again in a subsequent invocation.
*
* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
* Basically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
* Note that this is just a hint, it's always possible to any srcSize value.
* When a frame is fully decoded, @return will be 0.
* If decompression failed, @return is an error code which can be tested using LZ4F_isError().
*/
size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
@ -1097,7 +1109,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
{ LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);
{ LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); /* will change dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
}
break;
@ -1145,7 +1157,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
dctxPtr->tmpInSize += sizeToCopy;
if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */
if (dctxPtr->tmpInSize < BHSize) { /* not enough input for cBlockSize */
nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
doAnotherStage = 0;
break;
@ -1153,13 +1165,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
selectedIn = dctxPtr->tmpIn;
}
/* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */
/* case dstage_decodeCBlockSize: */ /* no more direct access, to remove scan-build warning */
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */
dctxPtr->dStage = dstage_getSuffix;
break;
}
if (nextCBlockSize > dctxPtr->maxBlockSize) return err0r(LZ4F_ERROR_GENERIC); /* invalid cBlockSize */
if (nextCBlockSize > dctxPtr->maxBlockSize)
return err0r(LZ4F_ERROR_maxBlockSize_invalid);
dctxPtr->tmpInTarget = nextCBlockSize;
if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
dctxPtr->dStage = dstage_copyDirect;
@ -1175,11 +1188,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
case dstage_copyDirect: /* uncompressed block */
{ size_t sizeToCopy = dctxPtr->tmpInTarget;
if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */
if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr;
if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
memcpy(dstPtr, srcPtr, sizeToCopy);
if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy;
if (dctxPtr->frameInfo.contentChecksumFlag)
XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
if (dctxPtr->frameInfo.contentSize)
dctxPtr->frameRemainingSize -= sizeToCopy;
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
@ -1240,10 +1255,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
else
decoder = LZ4F_decompress_safe;
decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
decodedSize = decoder((const char*)selectedIn, (char*)dstPtr,
(int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
(const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
if (dctxPtr->frameInfo.contentChecksumFlag)
XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
if (dctxPtr->frameInfo.contentSize)
dctxPtr->frameRemainingSize -= decodedSize;
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
@ -1280,10 +1299,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
}
/* Decode */
decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
if (decodedSize < 0) return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */
if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut,
(int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
(const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
if (decodedSize < 0)
return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */
if (dctxPtr->frameInfo.contentChecksumFlag)
XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
if (dctxPtr->frameInfo.contentSize)
dctxPtr->frameRemainingSize -= decodedSize;
dctxPtr->tmpOutSize = decodedSize;
dctxPtr->tmpOutStart = 0;
dctxPtr->dStage = dstage_flushOut;
@ -1314,7 +1338,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
case dstage_getSuffix:
{ size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
if (dctxPtr->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
if (dctxPtr->frameRemainingSize)
return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
if (suffixSize == 0) { /* frame completed */
nextSrcSizeHint = 0;
dctxPtr->dStage = dstage_getHeader;
@ -1375,7 +1400,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
dctxPtr->tmpInSize += sizeToCopy;
if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */
if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
/* not enough input to get full sBlockSize; wait for more */
nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
doAnotherStage = 0;
break;

8
lib/lz4frame.h
View File

@ -363,6 +363,14 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
const LZ4F_decompressOptions_t* dOptPtr);
/*! LZ4F_resetDecompressionContext() : v1.8.0
* In case of an error, the context is left in "undefined" state.
* In which case, it's necessary to reset it, before re-using it.
* This method can also be used to abruptly stop an unfinished decompression,
* and start a new with the same context. */
LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */
#if defined (__cplusplus)
}

9
lib/lz4frame_static.h
View File

@ -50,15 +50,6 @@ extern "C" {
#include "lz4frame.h"
/* --- Experimental functions --- */
/* LZ4F_resetDecompressionContext() :
* LZ4F_decompress() does not guarantee to leave dctx in clean state in case of errors.
* In order to re-use a dctx after a decompression error,
* use LZ4F_resetDecompressionContext() first.
* dctx will be able to start decompression on a new frame */
LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
/* --- Error List --- */
#define LZ4F_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) \

42
lib/lz4hc.c
View File

@ -69,9 +69,9 @@
/*=== Macros ===*/
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
@ -106,7 +106,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
U32 const h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - hashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
DELTANEXTU16(idx) = (U16)delta;
DELTANEXTU16(chainTable, idx) = (U16)delta;
hashTable[h] = idx;
idx++;
}
@ -115,8 +115,8 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
}
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
const BYTE* const ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
{
@ -138,8 +138,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
nbAttempts--;
if (matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (LZ4_read32(match) == LZ4_read32(ip)))
if ( (*(match+ml) == *(ip+ml)) /* can be longer */
&& (LZ4_read32(match) == LZ4_read32(ip)) )
{
size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
@ -156,7 +156,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
matchIndex -= DELTANEXTU16(matchIndex);
matchIndex -= DELTANEXTU16(chainTable, matchIndex);
}
return (int)ml;
@ -180,9 +180,9 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
const BYTE* const dictBase = hc4->dictBase;
U32 matchIndex;
int const delta = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
int delta = (int)(ip-iLowLimit);
U32 matchIndex;
/* First Match */
@ -206,14 +206,14 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
mlt -= back;
if (mlt > longest) {
longest = (int)mlt;
longest = mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
} } }
} else {
const BYTE* const matchPtr = dictBase + matchIndex;
if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
size_t mlt;
int mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
@ -222,10 +222,10 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
mlt -= back;
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
matchIndex -= DELTANEXTU16(matchIndex);
matchIndex -= DELTANEXTU16(chainTable, matchIndex);
}
return longest;
@ -238,12 +238,10 @@ typedef enum {
limitedDestSize = 2,
} limitedOutput_directive;
#define LZ4HC_DEBUG 0
#if LZ4HC_DEBUG
static unsigned debug = 0;
#ifndef LZ4HC_DEBUG
# define LZ4HC_DEBUG 0
#endif
/* LZ4HC_encodeSequence() :
* @return : 0 if ok,
* 1 if buffer issue detected */
@ -257,15 +255,15 @@ FORCE_INLINE int LZ4HC_encodeSequence (
BYTE* oend)
{
size_t length;
BYTE* token;
BYTE* const token = (*op)++;
#if LZ4HC_DEBUG
if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
printf("literal : %u -- match : %u -- offset : %u\n",
(U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
#endif
/* Encode Literal length */
length = (size_t)(*ip - *anchor);
token = (*op)++;
if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length >= RUN_MASK) {
size_t len = length - RUN_MASK;

26
programs/Makefile
View File

@ -51,7 +51,7 @@ CFLAGS ?= -O3
DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
LZ4_VERSION=$(LIBVER)
@ -75,28 +75,17 @@ all: lz4 lz4c
all32: CFLAGS+=-m32
all32: all
lz4: CFLAGS += $(DEBUGFLAGS)
lz4: $(OBJFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4-release: DEBUGFLAGS=
lz4-release: lz4
lz4c : CFLAGS += $(DEBUGFLAGS)
lz4c : $(SRCFILES)
$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
lz4c32: CFLAGS += -m32 $(DEBUGFLAGS)
lz4c32: $(SRCFILES)
lz4c32: CFLAGS += -m32
lz4c: CPPFLAGS += -DENABLE_LZ4C_LEGACY_OPTIONS
lz4c lz4c32 : $(SRCFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
@echo Cleaning completed
lz4.1: lz4.1.md
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
@ -108,6 +97,12 @@ clean-man:
preview-man: clean-man man
man ./lz4.1
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
@echo Cleaning completed
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
@ -120,7 +115,6 @@ unlz4: lz4
lz4cat: lz4
ln -s lz4 lz4cat
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
else

48
tests/frametest.c
View File

@ -40,6 +40,7 @@
#include <stdio.h> /* fprintf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
#include <assert.h>
#include "lz4frame_static.h"
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
@ -67,7 +68,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 256 KB;
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
@ -166,7 +166,7 @@ static unsigned FUZ_highbit(U32 v32)
*********************************************************/
int basicTests(U32 seed, double compressibility)
{
int testResult = 0;
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
void* const compressedBuffer = malloc(cBuffSize);
@ -176,9 +176,10 @@ int basicTests(U32 seed, double compressibility)
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
int basicTests_error = 0;
LZ4F_preferences_t prefs;
memset(&prefs, 0, sizeof(prefs));
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
goto _output_error;
@ -198,7 +199,7 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize);
DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
{ LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
@ -236,8 +237,6 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Decompression test : \n");
{ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedBufferSize = cSize;
BYTE* ip = (BYTE*)compressedBuffer;
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
@ -280,6 +279,7 @@ int basicTests(U32 seed, double compressibility)
{ size_t oSize = 0;
size_t iSize = 0;
LZ4F_frameInfo_t fi;
const BYTE* ip = (BYTE*)compressedBuffer;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
@ -290,7 +290,8 @@ int basicTests(U32 seed, double compressibility)
{ size_t nullSize = 0;
size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
LZ4F_getErrorName(fiError));
goto _output_error;
}
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
@ -314,10 +315,30 @@ int basicTests(U32 seed, double compressibility)
ip += iSize;
}
DISPLAYLEVEL(3, "Decode a buggy input : ");
assert(COMPRESSIBLE_NOISE_LENGTH > 64);
assert(cSize > 48);
memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */
memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */
{ size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
size_t cbSize = cSize;
size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
compressedBuffer, &cbSize,
NULL);
if (!LZ4F_isError(decompressError)) goto _output_error;
DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
}
memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */
DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
LZ4F_resetDecompressionContext(dCtx); /* always successful */
DISPLAYLEVEL(3, "Byte after byte : ");
{ BYTE* const ostart = (BYTE*)decodedBuffer;
BYTE* op = ostart;
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
const BYTE* ip = (const BYTE*) compressedBuffer;
const BYTE* const iend = ip + cSize;
while (ip < iend) {
size_t oSize = oend-op;
size_t iSize = 1;
@ -329,11 +350,7 @@ int basicTests(U32 seed, double compressibility)
{ U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH);
}
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
dCtx = NULL;
}
}
DISPLAYLEVEL(3, "Using 64 KB block : ");
@ -365,9 +382,6 @@ int basicTests(U32 seed, double compressibility)
const BYTE* ip = (const BYTE*)compressedBuffer;
const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
{ LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(createError)) goto _output_error; }
DISPLAYLEVEL(3, "random segment sizes : ");
while (ip < iend) {
unsigned const nbBits = FUZ_rand(&randState) % maxBits;
@ -551,10 +565,10 @@ _end:
free(decodedBuffer);
LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
LZ4F_freeCompressionContext(cctx); cctx = NULL;
return testResult;
return basicTests_error;
_output_error:
testResult = 1;
basicTests_error = 1;
DISPLAY("Error detected ! \n");
goto _end;
}