From 1efa48831e4e2422711262e5d3c3cbf613189d3b Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Tue, 2 May 2017 12:01:13 -0700
Subject: [PATCH 1/7] minor readability changes
---
lib/lz4hc.c | 42 ++++++++++++++++++++----------------------
1 file changed, 20 insertions(+), 22 deletions(-)
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 16fe029..b987d39 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -69,9 +69,9 @@
/*=== Macros ===*/
-#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
-#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
-#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
+#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
+#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
+#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
@@ -106,7 +106,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
U32 const h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - hashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
- DELTANEXTU16(idx) = (U16)delta;
+ DELTANEXTU16(chainTable, idx) = (U16)delta;
hashTable[h] = idx;
idx++;
}
@@ -115,8 +115,8 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
}
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */
- const BYTE* ip, const BYTE* const iLimit,
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
+ const BYTE* const ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
{
@@ -138,8 +138,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
nbAttempts--;
if (matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
- if (*(match+ml) == *(ip+ml)
- && (LZ4_read32(match) == LZ4_read32(ip)))
+ if ( (*(match+ml) == *(ip+ml)) /* can be longer */
+ && (LZ4_read32(match) == LZ4_read32(ip)) )
{
size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
@@ -156,7 +156,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
- matchIndex -= DELTANEXTU16(matchIndex);
+ matchIndex -= DELTANEXTU16(chainTable, matchIndex);
}
return (int)ml;
@@ -180,9 +180,9 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
const BYTE* const dictBase = hc4->dictBase;
- U32 matchIndex;
+ int const delta = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
- int delta = (int)(ip-iLowLimit);
+ U32 matchIndex;
/* First Match */
@@ -206,14 +206,14 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
mlt -= back;
if (mlt > longest) {
- longest = (int)mlt;
+ longest = mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
} } }
} else {
const BYTE* const matchPtr = dictBase + matchIndex;
if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
- size_t mlt;
+ int mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
@@ -222,10 +222,10 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
mlt -= back;
- if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
+ if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
- matchIndex -= DELTANEXTU16(matchIndex);
+ matchIndex -= DELTANEXTU16(chainTable, matchIndex);
}
return longest;
@@ -238,12 +238,10 @@ typedef enum {
limitedDestSize = 2,
} limitedOutput_directive;
-#define LZ4HC_DEBUG 0
-#if LZ4HC_DEBUG
-static unsigned debug = 0;
+#ifndef LZ4HC_DEBUG
+# define LZ4HC_DEBUG 0
#endif
-
/* LZ4HC_encodeSequence() :
* @return : 0 if ok,
* 1 if buffer issue detected */
@@ -257,15 +255,15 @@ FORCE_INLINE int LZ4HC_encodeSequence (
BYTE* oend)
{
size_t length;
- BYTE* token;
+ BYTE* const token = (*op)++;
#if LZ4HC_DEBUG
- if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
+ printf("literal : %u -- match : %u -- offset : %u\n",
+ (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
#endif
/* Encode Literal length */
length = (size_t)(*ip - *anchor);
- token = (*op)++;
if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length >= RUN_MASK) {
size_t len = length - RUN_MASK;
From fe932c4527aabd1c642b67fddf90e9be8ea380f8 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 10 May 2017 12:25:05 -0700
Subject: [PATCH 2/7] expose LZ4F_resetDecompressionContext()
---
lib/lz4frame.c | 134 +++++++++++++++++++++++++------------------------
lib/lz4frame.h | 11 ++++
2 files changed, 80 insertions(+), 65 deletions(-)
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 356be8a..7422638 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -209,7 +209,8 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
{
- LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
+ /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
+ LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
}
@@ -241,7 +242,8 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length)
/*-************************************
* Simple-pass compression functions
**************************************/
-static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize)
+static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
+ const size_t srcSize)
{
LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
size_t maxBlockSize = 64 KB;
@@ -256,11 +258,13 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
/* LZ4F_compressBound() :
* Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
- * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
+ * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
+ * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
* When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
-static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered)
+static size_t LZ4F_compressBound_internal(size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr,
+ size_t alreadyBuffered)
{
LZ4F_preferences_t prefsNull;
memset(&prefsNull, 0, sizeof(prefsNull));
@@ -298,14 +302,14 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
/*! LZ4F_compressFrame() :
-* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
-* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
-* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
-* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
-* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default.
-* The result of the function is the number of bytes written into dstBuffer.
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-*/
+ * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
+ * The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
+ * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
+ * Get the minimum value of dstCapacity by using LZ4F_compressFrameBound().
+ * The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default.
+ * The result of the function is the number of bytes written into dstBuffer.
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ */
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_cctx_t cctxI;
@@ -404,10 +408,10 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
/*! LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
- * @return : number of bytes written into dstBuffer for the header
- * or an error code (can be tested using LZ4F_isError())
+ * will write the frame header into dstBuffer.
+ * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+ * @return : number of bytes written into dstBuffer for the header
+ * or an error code (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr)
{
@@ -650,13 +654,13 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
/*! LZ4F_flush() :
-* Should you need to create compressed data immediately, without waiting for a block to be filled,
-* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
-* The result of the function is the number of bytes written into dstBuffer
-* (it can be zero, this means there was no data left within compressionContext)
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-*/
+ * Should you need to create compressed data immediately, without waiting for a block to be filled,
+ * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
+ * The result of the function is the number of bytes written into dstBuffer
+ * (it can be zero, this means there was no data left within compressionContext)
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ */
size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
@@ -687,14 +691,14 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
/*! LZ4F_compressEnd() :
-* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
-* It will flush whatever data remained within compressionContext (like LZ4_flush())
-* but also properly finalize the frame, with an endMark and a checksum.
-* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
-*/
+ * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
+ * It will flush whatever data remained within compressionContext (like LZ4_flush())
+ * but also properly finalize the frame, with an endMark and a checksum.
+ * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
+ */
size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
@@ -751,11 +755,11 @@ struct LZ4F_dctx_s {
/*! LZ4F_createDecompressionContext() :
-* Create a decompressionContext object, which will track all decompression operations.
-* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
-* Object can later be released using LZ4F_freeDecompressionContext().
-* @return : if != 0, there was an error during context creation.
-*/
+ * Create a decompressionContext object, which will track all decompression operations.
+ * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
+ * Object can later be released using LZ4F_freeDecompressionContext().
+ * @return : if != 0, there was an error during context creation.
+ */
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
@@ -802,9 +806,9 @@ LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
/*! LZ4F_headerSize() :
-* @return : size of frame header
-* or an error code, which can be tested using LZ4F_isError()
-*/
+ * @return : size of frame header
+ * or an error code, which can be tested using LZ4F_isError()
+ */
static size_t LZ4F_headerSize(const void* src, size_t srcSize)
{
/* minimal srcSize to determine header size */
@@ -825,13 +829,13 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
/*! LZ4F_decodeHeader() :
- input : `src` points at the **beginning of the frame**
- output : set internal values of dctx, such as
- dctxPtr->frameInfo and dctxPtr->dStage.
- Also allocates internal buffers.
- @return : nb Bytes read from src (necessarily <= srcSize)
- or an error code (testable with LZ4F_isError())
-*/
+ * input : `src` points at the **beginning of the frame**
+ * output : set internal values of dctx, such as
+ * dctxPtr->frameInfo and dctxPtr->dStage.
+ * Also allocates internal buffers.
+ * @return : nb Bytes read from src (necessarily <= srcSize)
+ * or an error code (testable with LZ4F_isError())
+ */
static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
{
unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID;
@@ -913,8 +917,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
/*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional. The objective is to provide relevant information for allocation purposes.
+ * This function extracts frame parameters (max blockSize, frame checksum, etc.).
+ * Usage is optional. Objective is to provide relevant information for allocation purposes.
* This function works in 2 situations :
* - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
* Amount of input data provided must be large enough to successfully decode the frame header.
@@ -1028,22 +1032,22 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi
/*! LZ4F_decompress() :
-* Call this function repetitively to regenerate data compressed within srcBuffer.
-* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
-*
-* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
-*
-* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
-* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
-* Remaining data will have to be presented again in a subsequent invocation.
-*
-* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
-* Basically, it's the size of the current (or remaining) compressed block + header of next block.
-* Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
-* Note that this is just a hint, it's always possible to any srcSize value.
-* When a frame is fully decoded, @return will be 0.
-* If decompression failed, @return is an error code which can be tested using LZ4F_isError().
-*/
+ * Call this function repetitively to regenerate data compressed within srcBuffer.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ *
+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ *
+ * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
+ * Remaining data will have to be presented again in a subsequent invocation.
+ *
+ * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
+ * Basically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
+ * Note that this is just a hint, it's always possible to any srcSize value.
+ * When a frame is fully decoded, @return will be 0.
+ * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
+ */
size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 2e79a17..b61fe7f 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -363,6 +363,17 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
const LZ4F_decompressOptions_t* dOptPtr);
+/*! LZ4F_resetDecompressionContext() :
+ * When decompression ends successfully,
+ * it's possible to start a new decompression immediately
+ * re-using the same context.
+ * However, in case of an error, the context is left in "undefined" state.
+ * In which case, it's necessary to reset it, before re-using it.
+ * This method can also be used to abruptly stop an unfinished decompression,
+ * and start a new on the same context. */
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+
+
#if defined (__cplusplus)
}
From b8575f2d2b1e05c20812215759b29bf2d0f7a708 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 10 May 2017 13:26:04 -0700
Subject: [PATCH 3/7] updated Makefile
to automatically build manual files with
make all
---
Makefile | 81 ++++++++++++++++-------------------
contrib/gen_manual/.gitignore | 2 +
contrib/gen_manual/Makefile | 26 +++++++++--
doc/lz4_manual.html | 18 ++++----
doc/lz4frame_manual.html | 75 ++++++++++++++++++++++----------
lib/Makefile | 13 +++---
lib/lz4frame.h | 2 +-
programs/Makefile | 26 +++++------
8 files changed, 140 insertions(+), 103 deletions(-)
create mode 100644 contrib/gen_manual/.gitignore
diff --git a/Makefile b/Makefile
index 1f5165d..24328d5 100644
--- a/Makefile
+++ b/Makefile
@@ -52,36 +52,36 @@ EXT =
endif
-.PHONY: default all lib lz4 clean test versionsTest examples
+.PHONY: default
+default: lib lz4-release
-default:
- @$(MAKE) -C $(LZ4DIR)
- @$(MAKE) -C $(PRGDIR)
- @cp $(PRGDIR)/lz4$(EXT) .
-
-all:
- @$(MAKE) -C $(LZ4DIR) $@
- @$(MAKE) -C $(PRGDIR) $@
- @$(MAKE) -C $(TESTDIR) $@
- @$(MAKE) -C $(EXDIR) $@
+.PHONY: all
+all: default examples manuals
+.PHONY: lib
lib:
@$(MAKE) -C $(LZ4DIR)
-lz4:
+.PHONY: lz4 lz4-release
+lz4 lz4-release: lib
@$(MAKE) -C $(PRGDIR) $@
@cp $(PRGDIR)/lz4$(EXT) .
-lz4-release:
- @$(MAKE) -C $(PRGDIR)
- @cp $(PRGDIR)/lz4$(EXT) .
+.PHONY: examples
+examples: lib lz4-release
+ $(MAKE) -C $(EXDIR) test
+.PHONY: manuals
+manuals:
+ @$(MAKE) -C contrib/gen_manual $@
+
+.PHONY: clean
clean:
+ @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
- @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(EXDIR) $@ > $(VOID)
- @$(MAKE) -C examples $@ > $(VOID)
+ @$(MAKE) -C contrib/gen_manual $@
@$(RM) lz4$(EXT)
@echo Cleaning completed
@@ -92,17 +92,31 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX
-install:
- @$(MAKE) -C $(LZ4DIR) $@
- @$(MAKE) -C $(PRGDIR) $@
-
-uninstall:
+install uninstall:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
travis-install:
$(MAKE) -j1 install PREFIX=~/install_test_dir
+cmake:
+ @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
+
+endif
+
+
+ifneq (,$(filter MSYS%,$(shell uname)))
+HOST_OS = MSYS
+CMAKE_PARAMS = -G"MSYS Makefiles"
+endif
+
+
+#------------------------------------------------------------------------
+#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+#------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+
+.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
@@ -135,31 +149,10 @@ platformTest: clean
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all
$(MAKE) -C $(TESTDIR) test-platform
+.PHONY: versionsTest
versionsTest: clean
$(MAKE) -C $(TESTDIR) $@
-examples:
- $(MAKE) -C $(LZ4DIR)
- $(MAKE) -C $(PRGDIR) lz4
- $(MAKE) -C examples test
-
-endif
-
-
-ifneq (,$(filter MSYS%,$(shell uname)))
-HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles"
-endif
-
-
-#------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
-#------------------------------------------------------------------------
-ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-
-cmake:
- @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
-
gpptest: clean
g++ -v
CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
diff --git a/contrib/gen_manual/.gitignore b/contrib/gen_manual/.gitignore
new file mode 100644
index 0000000..6ea967f
--- /dev/null
+++ b/contrib/gen_manual/.gitignore
@@ -0,0 +1,2 @@
+# build artefact
+gen_manual
diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile
index adbcca2..9fbe858 100644
--- a/contrib/gen_manual/Makefile
+++ b/contrib/gen_manual/Makefile
@@ -35,7 +35,15 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -W
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
-
+LZ4API = ../../lib/lz4.h
+LZ4MANUAL = ../../doc/lz4_manual.html
+LZ4FAPI = ../../lib/lz4frame.h
+LZ4FMANUAL = ../../doc/lz4frame_manual.html
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LZ4VER := $(shell echo $(LIBVER_SCRIPT))
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@@ -45,14 +53,24 @@ EXT =
endif
-.PHONY: default gen_manual
-
+.PHONY: default
default: gen_manual
gen_manual: gen_manual.cpp
- $(CXX) $(FLAGS) $^ -o $@$(EXT)
+ $(CXX) $(FLAGS) $^ -o $@$(EXT)
+$(LZ4MANUAL) : gen_manual $(LZ4API)
+ echo "Update lz4 manual in /doc"
+ ./gen_manual $(LZ4VER) $(LZ4API) $@
+$(LZ4FMANUAL) : gen_manual $(LZ4FAPI)
+ echo "Update lz4frame manual in /doc"
+ ./gen_manual $(LZ4VER) $(LZ4FAPI) $@
+
+.PHONY: manuals
+manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL)
+
+.PHONY: clean
clean:
@$(RM) gen_manual$(EXT)
@echo Cleaning completed
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ff6e149..4c4174d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
-lz4 1.7.5 Manual
+1.7.6 Manual
-lz4 1.7.5 Manual
+1.7.6 Manual
Contents
@@ -170,21 +170,21 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr);
-int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
- Important : Previous data blocks are assumed to still be present and unmodified !
+ Important : Previous data blocks are assumed to remain present and unmodified !
'dst' buffer must be already allocated.
- If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ After an error, the stream status is invalid, and it can only be reset or freed.
int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
- If previously compressed data block is not guaranteed to remain available at its memory location,
+
If previously compressed data block is not guaranteed to remain available at its current memory location,
save it into a safer place (char* safeBuffer).
- Note : you don't need to call LZ4_loadDict() afterwards,
- dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
- Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
+ @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index f76ec3d..5d553ed 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,19 +1,23 @@
-lz4frame 1.7.5 Manual
+1.7.6 Manual
-lz4frame 1.7.5 Manual
+1.7.6 Manual
Contents
- Introduction
-- Error management
-- Frame compression types
-- Simple compression function
-- Advanced compression functions
-- Decompression functions
+- Compiler specifics
+- Error management
+- Frame compression types
+- Simple compression function
+- Advanced compression functions
+- Resource Management
+- Compression
+- Decompression functions
+- Streaming decompression functions
Introduction
@@ -22,13 +26,15 @@
of encoding standard metadata alongside LZ4-compressed blocks.
-Error management
+Compiler specifics
+
+Error management
unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */
const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */
-Frame compression types
+Frame compression types
typedef enum {
LZ4F_default=0,
@@ -77,7 +83,7 @@
typedef struct {
LZ4F_frameInfo_t frameInfo;
- int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
+ int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
@@ -86,7 +92,7 @@
All reserved fields must be set to zero.
-Simple compression function
+Simple compression function
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
@@ -105,17 +111,19 @@
-Advanced compression functions
+Advanced compression functions
typedef struct {
unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */
unsigned reserved[3];
} LZ4F_compressOptions_t;
+Resource Management
+
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
The function will provide a pointer to a fully allocated LZ4F_cctx object.
If @return != zero, there was an error during context creation.
@@ -123,6 +131,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
+Compression
+
size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
will write the frame header into dstBuffer.
dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
@@ -173,7 +183,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
-Decompression functions
+Decompression functions
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
@@ -181,7 +191,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
} LZ4F_decompressOptions_t;
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
@@ -192,19 +202,27 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
+Streaming decompression functions
+
size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
- This function decodes frame header information (such as max blockSize, frame checksum, etc.).
- Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
- A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
+
This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
+ Its usage is optional. Extracted information can be useful for allocation purposes, typically.
+ This function works in 2 situations :
+ - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+ Input size must be large enough to successfully decode the entire frame header.
+ Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
+ It's allowed to provide more input data than this minimum.
+ - After decoding has been started.
+ In which case, no input is read, frame parameters are extracted from dctx.
+ If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- Decompression must resume from this point (srcBuffer + *srcSizePtr).
- Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
- Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
+ Decompression must resume from (srcBuffer + *srcSizePtr).
@return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
or an error code which can be tested using LZ4F_isError()
- (typically, when there is not enough src bytes to fully decode the frame header)
+ note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
@@ -227,14 +245,25 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
@return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
+ Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
This is just a hint though, it's always possible to provide any srcSize.
When a frame is fully decoded, @return will be 0 (no more data expected).
If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
After a frame is fully decoded, dctx can be used again to decompress another frame.
+ After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
+LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+ When decompression ends successfully,
+ it's possible to start a new decompression immediately
+ re-using the same context.
+ However, in case of an error, the context is left in "undefined" state.
+ In which case, it's necessary to reset it, before re-using it.
+ This method can also be used to abruptly stop an unfinished decompression,
+ and start a new on the same context.
+
+
diff --git a/lib/Makefile b/lib/Makefile
index 9a794b8..c6fd7b8 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -46,10 +46,10 @@ BUILD_STATIC:= yes
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
-DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith -Wstrict-aliasing=1
-CFLAGS += $(MOREFLAGS)
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@@ -70,12 +70,13 @@ endif
LIBLZ4 = liblz4.$(SHARED_EXT_VER)
+.PHONY: default
default: lib-release
-lib-release: liblz4.a liblz4
+lib-release: DEBUGFLAGS :=
+lib-release: lib
-lib: CFLAGS += $(DEBUGFLAGS)
-lib: lib-release
+lib: liblz4.a liblz4
all: lib
@@ -83,7 +84,7 @@ all32: CFLAGS+=-m32
all32: all
liblz4.a: *.c
-ifeq ($(BUILD_STATIC),yes)
+ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
@$(AR) rcs $@ *.o
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index b61fe7f..9a2130a 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -363,7 +363,7 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
const LZ4F_decompressOptions_t* dOptPtr);
-/*! LZ4F_resetDecompressionContext() :
+/*! LZ4F_resetDecompressionContext() : v1.8.0
* When decompression ends successfully,
* it's possible to start a new decompression immediately
* re-using the same context.
diff --git a/programs/Makefile b/programs/Makefile
index 4a8103c..2e73780 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -51,7 +51,7 @@ CFLAGS ?= -O3
DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
-CFLAGS += $(MOREFLAGS)
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
LZ4_VERSION=$(LIBVER)
@@ -75,28 +75,17 @@ all: lz4 lz4c
all32: CFLAGS+=-m32
all32: all
-lz4: CFLAGS += $(DEBUGFLAGS)
lz4: $(OBJFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4-release: DEBUGFLAGS=
lz4-release: lz4
-lz4c : CFLAGS += $(DEBUGFLAGS)
-lz4c : $(SRCFILES)
- $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
-
-lz4c32: CFLAGS += -m32 $(DEBUGFLAGS)
-lz4c32: $(SRCFILES)
+lz4c32: CFLAGS += -m32
+lz4c: CPPFLAGS += -DENABLE_LZ4C_LEGACY_OPTIONS
+lz4c lz4c32 : $(SRCFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
-clean:
- @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
- @$(RM) core *.o *.test tmp* \
- lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
- @echo Cleaning completed
-
-
lz4.1: lz4.1.md
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
@@ -108,6 +97,12 @@ clean-man:
preview-man: clean-man man
man ./lz4.1
+clean:
+ @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
+ @$(RM) core *.o *.test tmp* \
+ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
+ @echo Cleaning completed
+
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
@@ -120,7 +115,6 @@ unlz4: lz4
lz4cat: lz4
ln -s lz4 lz4cat
-
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
else
From a902127613921408bb02b1debe9371e9bcfaf9f2 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 10 May 2017 13:27:18 -0700
Subject: [PATCH 4/7] bumped version number to 1.8.0
due to addition of prototype
LZ4F_resetDecompressionContext()
---
doc/lz4_manual.html | 4 ++--
doc/lz4frame_manual.html | 4 ++--
lib/lz4.h | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 4c4174d..fecd8cd 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
-1.7.6 Manual
+1.8.0 Manual
-1.7.6 Manual
+1.8.0 Manual
Contents
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 5d553ed..fa2c0b9 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
-1.7.6 Manual
+1.8.0 Manual
-1.7.6 Manual
+1.8.0 Manual
Contents
diff --git a/lib/lz4.h b/lib/lz4.h
index d5a3016..5b6bc92 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -88,8 +88,8 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
-#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
From 2012e4de9e2c33a5abb34d40082afc8918852c6e Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 10 May 2017 14:51:09 -0700
Subject: [PATCH 5/7] fixed c_standards tests
and added entry "make list"
---
Makefile | 24 ++++++++++++++----------
tests/frametest.c | 3 ++-
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/Makefile b/Makefile
index 24328d5..b31ebd0 100644
--- a/Makefile
+++ b/Makefile
@@ -56,7 +56,10 @@ endif
default: lib lz4-release
.PHONY: all
-all: default examples manuals
+all: allmost manuals
+
+.PHONY: allmost
+allmost: lib lz4 examples
.PHONY: lib
lib:
@@ -68,7 +71,7 @@ lz4 lz4-release: lib
@cp $(PRGDIR)/lz4$(EXT) .
.PHONY: examples
-examples: lib lz4-release
+examples: lib lz4
$(MAKE) -C $(EXDIR) test
.PHONY: manuals
@@ -116,6 +119,10 @@ endif
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+.PHONY: list
+list:
+ @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
@@ -166,13 +173,10 @@ gpptest32: clean
CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
c_standards: clean
- $(MAKE) all MOREFLAGS="-std=gnu90 -Werror"
- $(MAKE) clean
- $(MAKE) all MOREFLAGS="-std=c99 -Werror"
- $(MAKE) clean
- $(MAKE) all MOREFLAGS="-std=gnu99 -Werror"
- $(MAKE) clean
- $(MAKE) all MOREFLAGS="-std=c11 -Werror"
- $(MAKE) clean
+ # note : lz4 is not C90 compatible, because it requires long long support
+ CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
+ CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost
+ CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost
+ CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost
endif
diff --git a/tests/frametest.c b/tests/frametest.c
index a4c548e..04597a4 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -290,7 +290,8 @@ int basicTests(U32 seed, double compressibility)
{ size_t nullSize = 0;
size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
- DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
+ DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
+ LZ4F_getErrorName(fiError));
goto _output_error;
}
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
From e60cbb5cacaa43eda7a3df7c2d9051a9437e7046 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 10 May 2017 16:28:36 -0700
Subject: [PATCH 6/7] added test for LZ4F_resetDecompressionContext()
---
doc/lz4frame_manual.html | 9 ++----
lib/lz4frame.c | 66 ++++++++++++++++++++++++++--------------
lib/lz4frame.h | 9 ++----
lib/lz4frame_static.h | 9 ------
tests/frametest.c | 45 +++++++++++++++++----------
5 files changed, 79 insertions(+), 59 deletions(-)
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index fa2c0b9..87750a1 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -255,14 +255,11 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
-LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
- When decompression ends successfully,
- it's possible to start a new decompression immediately
- re-using the same context.
- However, in case of an error, the context is left in "undefined" state.
+
void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */
+ In case of an error, the context is left in "undefined" state.
In which case, it's necessary to reset it, before re-using it.
This method can also be used to abruptly stop an unfinished decompression,
- and start a new on the same context.
+ and start a new with the same context.
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7422638..fb37789 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -798,10 +798,9 @@ typedef enum {
dstage_skipSkippable
} dStage_t;
-LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
+void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
{
dctx->dStage = dstage_getHeader;
- return 0;
}
@@ -939,7 +938,8 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
size_t o=0, i=0;
*srcSizePtr = 0;
*frameInfoPtr = dctxPtr->frameInfo;
- return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */
+ /* returns : recommended nb of bytes for LZ4F_decompress() */
+ return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);
} else {
if (dctxPtr->dStage == dstage_storeHeader) {
/* frame decoding already started, in the middle of header => automatic fail */
@@ -949,7 +949,10 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
size_t decodeResult;
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
- if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); }
+ if (*srcSizePtr < hSize) {
+ *srcSizePtr=0;
+ return err0r(LZ4F_ERROR_frameHeader_incomplete);
+ }
decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize);
if (LZ4F_isError(decodeResult)) {
@@ -965,10 +968,15 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
/* trivial redirector, for common prototype */
-static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
+static int LZ4F_decompress_safe (const char* src,
+ char* dst,
+ int compressedSize,
+ int dstCapacity,
+ const char* dictStart,
+ int dictSize)
{
(void)dictStart; (void)dictSize;
- return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
+ return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity);
}
@@ -1101,7 +1109,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
- { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);
+ { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); /* will change dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
}
break;
@@ -1149,7 +1157,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */
+ if (dctxPtr->tmpInSize < BHSize) { /* not enough input for cBlockSize */
nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
doAnotherStage = 0;
break;
@@ -1157,13 +1165,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
selectedIn = dctxPtr->tmpIn;
}
- /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */
+ /* case dstage_decodeCBlockSize: */ /* no more direct access, to remove scan-build warning */
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */
dctxPtr->dStage = dstage_getSuffix;
break;
}
- if (nextCBlockSize > dctxPtr->maxBlockSize) return err0r(LZ4F_ERROR_GENERIC); /* invalid cBlockSize */
+ if (nextCBlockSize > dctxPtr->maxBlockSize)
+ return err0r(LZ4F_ERROR_maxBlockSize_invalid);
dctxPtr->tmpInTarget = nextCBlockSize;
if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
dctxPtr->dStage = dstage_copyDirect;
@@ -1179,11 +1188,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
case dstage_copyDirect: /* uncompressed block */
{ size_t sizeToCopy = dctxPtr->tmpInTarget;
- if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */
+ if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr;
if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
memcpy(dstPtr, srcPtr, sizeToCopy);
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
- if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy;
+ if (dctxPtr->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
+ if (dctxPtr->frameInfo.contentSize)
+ dctxPtr->frameRemainingSize -= sizeToCopy;
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
@@ -1244,10 +1255,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
else
decoder = LZ4F_decompress_safe;
- decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+ decodedSize = decoder((const char*)selectedIn, (char*)dstPtr,
+ (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
+ (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
- if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
+ if (dctxPtr->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
+ if (dctxPtr->frameInfo.contentSize)
+ dctxPtr->frameRemainingSize -= decodedSize;
/* dictionary management */
if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
@@ -1284,10 +1299,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
}
/* Decode */
- decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
- if (decodedSize < 0) return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
- if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
+ decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut,
+ (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
+ (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+ if (decodedSize < 0)
+ return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */
+ if (dctxPtr->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
+ if (dctxPtr->frameInfo.contentSize)
+ dctxPtr->frameRemainingSize -= decodedSize;
dctxPtr->tmpOutSize = decodedSize;
dctxPtr->tmpOutStart = 0;
dctxPtr->dStage = dstage_flushOut;
@@ -1318,7 +1338,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
case dstage_getSuffix:
{ size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
- if (dctxPtr->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
+ if (dctxPtr->frameRemainingSize)
+ return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
if (suffixSize == 0) { /* frame completed */
nextSrcSizeHint = 0;
dctxPtr->dStage = dstage_getHeader;
@@ -1379,7 +1400,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */
+ if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
+ /* not enough input to get full sBlockSize; wait for more */
nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
doAnotherStage = 0;
break;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9a2130a..b1719e2 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -364,14 +364,11 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
/*! LZ4F_resetDecompressionContext() : v1.8.0
- * When decompression ends successfully,
- * it's possible to start a new decompression immediately
- * re-using the same context.
- * However, in case of an error, the context is left in "undefined" state.
+ * In case of an error, the context is left in "undefined" state.
* In which case, it's necessary to reset it, before re-using it.
* This method can also be used to abruptly stop an unfinished decompression,
- * and start a new on the same context. */
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+ * and start a new with the same context. */
+LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index 8ea496d..d3bae82 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -50,15 +50,6 @@ extern "C" {
#include "lz4frame.h"
-/* --- Experimental functions --- */
-/* LZ4F_resetDecompressionContext() :
- * LZ4F_decompress() does not guarantee to leave dctx in clean state in case of errors.
- * In order to re-use a dctx after a decompression error,
- * use LZ4F_resetDecompressionContext() first.
- * dctx will be able to start decompression on a new frame */
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
-
-
/* --- Error List --- */
#define LZ4F_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) \
diff --git a/tests/frametest.c b/tests/frametest.c
index 04597a4..538bedc 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -40,6 +40,7 @@
#include /* fprintf */
#include /* strcmp */
#include /* clock_t, clock(), CLOCKS_PER_SEC */
+#include
#include "lz4frame_static.h"
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
@@ -67,7 +68,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 256 KB;
-#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
@@ -166,7 +166,7 @@ static unsigned FUZ_highbit(U32 v32)
*********************************************************/
int basicTests(U32 seed, double compressibility)
{
- int testResult = 0;
+#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
void* const compressedBuffer = malloc(cBuffSize);
@@ -176,9 +176,10 @@ int basicTests(U32 seed, double compressibility)
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
-
+ int basicTests_error = 0;
LZ4F_preferences_t prefs;
memset(&prefs, 0, sizeof(prefs));
+
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
goto _output_error;
@@ -198,7 +199,7 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize);
+ DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
{ LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
@@ -236,8 +237,6 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Decompression test : \n");
{ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedBufferSize = cSize;
- BYTE* ip = (BYTE*)compressedBuffer;
- BYTE* const iend = (BYTE*)compressedBuffer + cSize;
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
@@ -280,6 +279,7 @@ int basicTests(U32 seed, double compressibility)
{ size_t oSize = 0;
size_t iSize = 0;
LZ4F_frameInfo_t fi;
+ const BYTE* ip = (BYTE*)compressedBuffer;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
@@ -315,10 +315,30 @@ int basicTests(U32 seed, double compressibility)
ip += iSize;
}
+ DISPLAYLEVEL(3, "Decode a buggy input : ");
+ assert(COMPRESSIBLE_NOISE_LENGTH > 64);
+ assert(cSize > 48);
+ memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */
+ memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */
+ { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t cbSize = cSize;
+ size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
+ compressedBuffer, &cbSize,
+ NULL);
+ if (!LZ4F_isError(decompressError)) goto _output_error;
+ DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
+ }
+ memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */
+
+ DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
+ LZ4F_resetDecompressionContext(dCtx); /* always successful */
+
DISPLAYLEVEL(3, "Byte after byte : ");
{ BYTE* const ostart = (BYTE*)decodedBuffer;
BYTE* op = ostart;
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
+ const BYTE* ip = compressedBuffer;
+ const BYTE* const iend = ip + cSize;
while (ip < iend) {
size_t oSize = oend-op;
size_t iSize = 1;
@@ -330,11 +350,7 @@ int basicTests(U32 seed, double compressibility)
{ U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH);
- }
-
- errorCode = LZ4F_freeDecompressionContext(dCtx);
- if (LZ4F_isError(errorCode)) goto _output_error;
- dCtx = NULL;
+ }
}
DISPLAYLEVEL(3, "Using 64 KB block : ");
@@ -366,9 +382,6 @@ int basicTests(U32 seed, double compressibility)
const BYTE* ip = (const BYTE*)compressedBuffer;
const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
- { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
- if (LZ4F_isError(createError)) goto _output_error; }
-
DISPLAYLEVEL(3, "random segment sizes : ");
while (ip < iend) {
unsigned const nbBits = FUZ_rand(&randState) % maxBits;
@@ -552,10 +565,10 @@ _end:
free(decodedBuffer);
LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
LZ4F_freeCompressionContext(cctx); cctx = NULL;
- return testResult;
+ return basicTests_error;
_output_error:
- testResult = 1;
+ basicTests_error = 1;
DISPLAY("Error detected ! \n");
goto _end;
}
From 2600a154bebdf79637e289ed19e53e2c72a168d7 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 10 May 2017 17:06:31 -0700
Subject: [PATCH 7/7] fix (minor) g++ compatibility for frametest
---
tests/frametest.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/frametest.c b/tests/frametest.c
index 538bedc..0dadf9f 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -337,7 +337,7 @@ int basicTests(U32 seed, double compressibility)
{ BYTE* const ostart = (BYTE*)decodedBuffer;
BYTE* op = ostart;
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
- const BYTE* ip = compressedBuffer;
+ const BYTE* ip = (const BYTE*) compressedBuffer;
const BYTE* const iend = ip + cSize;
while (ip < iend) {
size_t oSize = oend-op;