From 1efa48831e4e2422711262e5d3c3cbf613189d3b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 May 2017 12:01:13 -0700 Subject: [PATCH 1/7] minor readability changes --- lib/lz4hc.c | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 16fe029..b987d39 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -69,9 +69,9 @@ /*=== Macros ===*/ -#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ -#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -106,7 +106,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) U32 const h = LZ4HC_hashPtr(base+idx); size_t delta = idx - hashTable[h]; if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - DELTANEXTU16(idx) = (U16)delta; + DELTANEXTU16(chainTable, idx) = (U16)delta; hashTable[h] = idx; idx++; } @@ -115,8 +115,8 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) } -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, const BYTE** matchpos, const int maxNbAttempts) { @@ -138,8 +138,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In nbAttempts--; if (matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; - if (*(match+ml) == *(ip+ml) - && (LZ4_read32(match) == LZ4_read32(ip))) + if ( (*(match+ml) == *(ip+ml)) /* can be longer */ + && (LZ4_read32(match) == LZ4_read32(ip)) ) { size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; if (mlt > ml) { ml = mlt; *matchpos = match; } @@ -156,7 +156,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ } } - matchIndex -= DELTANEXTU16(matchIndex); + matchIndex -= DELTANEXTU16(chainTable, matchIndex); } return (int)ml; @@ -180,9 +180,9 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( const BYTE* const lowPrefixPtr = base + dictLimit; const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); const BYTE* const dictBase = hc4->dictBase; - U32 matchIndex; + int const delta = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; - int delta = (int)(ip-iLowLimit); + U32 matchIndex; /* First Match */ @@ -206,14 +206,14 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( mlt -= back; if (mlt > longest) { - longest = (int)mlt; + longest = mlt; *matchpos = matchPtr+back; *startpos = ip+back; } } } } else { const BYTE* const matchPtr = dictBase + matchIndex; if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { - size_t mlt; + int mlt; int back=0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; @@ -222,10 +222,10 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; mlt -= back; - if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } + if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } } - matchIndex -= DELTANEXTU16(matchIndex); + matchIndex -= DELTANEXTU16(chainTable, matchIndex); } return longest; @@ -238,12 +238,10 @@ typedef enum { limitedDestSize = 2, } limitedOutput_directive; -#define LZ4HC_DEBUG 0 -#if LZ4HC_DEBUG -static unsigned debug = 0; +#ifndef LZ4HC_DEBUG +# define LZ4HC_DEBUG 0 #endif - /* LZ4HC_encodeSequence() : * @return : 0 if ok, * 1 if buffer issue detected */ @@ -257,15 +255,15 @@ FORCE_INLINE int LZ4HC_encodeSequence ( BYTE* oend) { size_t length; - BYTE* token; + BYTE* const token = (*op)++; #if LZ4HC_DEBUG - if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); + printf("literal : %u -- match : %u -- offset : %u\n", + (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); #endif /* Encode Literal length */ length = (size_t)(*ip - *anchor); - token = (*op)++; if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ if (length >= RUN_MASK) { size_t len = length - RUN_MASK; From fe932c4527aabd1c642b67fddf90e9be8ea380f8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 May 2017 12:25:05 -0700 Subject: [PATCH 2/7] expose LZ4F_resetDecompressionContext() --- lib/lz4frame.c | 134 +++++++++++++++++++++++++------------------------ lib/lz4frame.h | 11 ++++ 2 files changed, 80 insertions(+), 65 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 356be8a..7422638 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -209,7 +209,8 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) { - LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ + /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ + LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); return (LZ4F_errorCode_t)-(ptrdiff_t)code; } @@ -241,7 +242,8 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length) /*-************************************ * Simple-pass compression functions **************************************/ -static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) +static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, + const size_t srcSize) { LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; size_t maxBlockSize = 64 KB; @@ -256,11 +258,13 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI /* LZ4F_compressBound() : * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. - * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. - * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ -static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) +static size_t LZ4F_compressBound_internal(size_t srcSize, + const LZ4F_preferences_t* preferencesPtr, + size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); @@ -298,14 +302,14 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere /*! LZ4F_compressFrame() : -* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() -* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) -* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. -* The result of the function is the number of bytes written into dstBuffer. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ + * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. + * The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. + * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) + * Get the minimum value of dstCapacity by using LZ4F_compressFrameBound(). + * The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default. + * The result of the function is the number of bytes written into dstBuffer. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_t cctxI; @@ -404,10 +408,10 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. - * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) */ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) { @@ -650,13 +654,13 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci /*! LZ4F_flush() : -* Should you need to create compressed data immediately, without waiting for a block to be filled, -* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. -* The result of the function is the number of bytes written into dstBuffer -* (it can be zero, this means there was no data left within compressionContext) -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -*/ + * Should you need to create compressed data immediately, without waiting for a block to be filled, + * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within compressionContext) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + */ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; @@ -687,14 +691,14 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const /*! LZ4F_compressEnd() : -* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). -* It will flush whatever data remained within compressionContext (like LZ4_flush()) -* but also properly finalize the frame, with an endMark and a checksum. -* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. -*/ + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. + */ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; @@ -751,11 +755,11 @@ struct LZ4F_dctx_s { /*! LZ4F_createDecompressionContext() : -* Create a decompressionContext object, which will track all decompression operations. -* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. -* Object can later be released using LZ4F_freeDecompressionContext(). -* @return : if != 0, there was an error during context creation. -*/ + * Create a decompressionContext object, which will track all decompression operations. + * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * Object can later be released using LZ4F_freeDecompressionContext(). + * @return : if != 0, there was an error during context creation. + */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); @@ -802,9 +806,9 @@ LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) /*! LZ4F_headerSize() : -* @return : size of frame header -* or an error code, which can be tested using LZ4F_isError() -*/ + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + */ static size_t LZ4F_headerSize(const void* src, size_t srcSize) { /* minimal srcSize to determine header size */ @@ -825,13 +829,13 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) /*! LZ4F_decodeHeader() : - input : `src` points at the **beginning of the frame** - output : set internal values of dctx, such as - dctxPtr->frameInfo and dctxPtr->dStage. - Also allocates internal buffers. - @return : nb Bytes read from src (necessarily <= srcSize) - or an error code (testable with LZ4F_isError()) -*/ + * input : `src` points at the **beginning of the frame** + * output : set internal values of dctx, such as + * dctxPtr->frameInfo and dctxPtr->dStage. + * Also allocates internal buffers. + * @return : nb Bytes read from src (necessarily <= srcSize) + * or an error code (testable with LZ4F_isError()) + */ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize) { unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID; @@ -913,8 +917,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS /*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (such as max blockSize, frame checksum, etc.). - * Its usage is optional. The objective is to provide relevant information for allocation purposes. + * This function extracts frame parameters (max blockSize, frame checksum, etc.). + * Usage is optional. Objective is to provide relevant information for allocation purposes. * This function works in 2 situations : * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. * Amount of input data provided must be large enough to successfully decode the frame header. @@ -1028,22 +1032,22 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi /*! LZ4F_decompress() : -* Call this function repetitively to regenerate data compressed within srcBuffer. -* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. -* -* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). -* -* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. -* Remaining data will have to be presented again in a subsequent invocation. -* -* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. -* Basically, it's the size of the current (or remaining) compressed block + header of next block. -* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. -* Note that this is just a hint, it's always possible to any srcSize value. -* When a frame is fully decoded, @return will be 0. -* If decompression failed, @return is an error code which can be tested using LZ4F_isError(). -*/ + * Call this function repetitively to regenerate data compressed within srcBuffer. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. + * Remaining data will have to be presented again in a subsequent invocation. + * + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Basically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, it's always possible to any srcSize value. + * When a frame is fully decoded, @return will be 0. + * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). + */ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 2e79a17..b61fe7f 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -363,6 +363,17 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, const LZ4F_decompressOptions_t* dOptPtr); +/*! LZ4F_resetDecompressionContext() : + * When decompression ends successfully, + * it's possible to start a new decompression immediately + * re-using the same context. + * However, in case of an error, the context is left in "undefined" state. + * In which case, it's necessary to reset it, before re-using it. + * This method can also be used to abruptly stop an unfinished decompression, + * and start a new on the same context. */ +LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); + + #if defined (__cplusplus) } From b8575f2d2b1e05c20812215759b29bf2d0f7a708 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 May 2017 13:26:04 -0700 Subject: [PATCH 3/7] updated Makefile to automatically build manual files with make all --- Makefile | 81 ++++++++++++++++------------------- contrib/gen_manual/.gitignore | 2 + contrib/gen_manual/Makefile | 26 +++++++++-- doc/lz4_manual.html | 18 ++++---- doc/lz4frame_manual.html | 75 ++++++++++++++++++++++---------- lib/Makefile | 13 +++--- lib/lz4frame.h | 2 +- programs/Makefile | 26 +++++------ 8 files changed, 140 insertions(+), 103 deletions(-) create mode 100644 contrib/gen_manual/.gitignore diff --git a/Makefile b/Makefile index 1f5165d..24328d5 100644 --- a/Makefile +++ b/Makefile @@ -52,36 +52,36 @@ EXT = endif -.PHONY: default all lib lz4 clean test versionsTest examples +.PHONY: default +default: lib lz4-release -default: - @$(MAKE) -C $(LZ4DIR) - @$(MAKE) -C $(PRGDIR) - @cp $(PRGDIR)/lz4$(EXT) . - -all: - @$(MAKE) -C $(LZ4DIR) $@ - @$(MAKE) -C $(PRGDIR) $@ - @$(MAKE) -C $(TESTDIR) $@ - @$(MAKE) -C $(EXDIR) $@ +.PHONY: all +all: default examples manuals +.PHONY: lib lib: @$(MAKE) -C $(LZ4DIR) -lz4: +.PHONY: lz4 lz4-release +lz4 lz4-release: lib @$(MAKE) -C $(PRGDIR) $@ @cp $(PRGDIR)/lz4$(EXT) . -lz4-release: - @$(MAKE) -C $(PRGDIR) - @cp $(PRGDIR)/lz4$(EXT) . +.PHONY: examples +examples: lib lz4-release + $(MAKE) -C $(EXDIR) test +.PHONY: manuals +manuals: + @$(MAKE) -C contrib/gen_manual $@ + +.PHONY: clean clean: + @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(TESTDIR) $@ > $(VOID) - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(EXDIR) $@ > $(VOID) - @$(MAKE) -C examples $@ > $(VOID) + @$(MAKE) -C contrib/gen_manual $@ @$(RM) lz4$(EXT) @echo Cleaning completed @@ -92,17 +92,31 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) HOST_OS = POSIX -install: - @$(MAKE) -C $(LZ4DIR) $@ - @$(MAKE) -C $(PRGDIR) $@ - -uninstall: +install uninstall: @$(MAKE) -C $(LZ4DIR) $@ @$(MAKE) -C $(PRGDIR) $@ travis-install: $(MAKE) -j1 install PREFIX=~/install_test_dir +cmake: + @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE) + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) + +.PHONY: test test: $(MAKE) -C $(TESTDIR) $@ @@ -135,31 +149,10 @@ platformTest: clean CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all $(MAKE) -C $(TESTDIR) test-platform +.PHONY: versionsTest versionsTest: clean $(MAKE) -C $(TESTDIR) $@ -examples: - $(MAKE) -C $(LZ4DIR) - $(MAKE) -C $(PRGDIR) lz4 - $(MAKE) -C examples test - -endif - - -ifneq (,$(filter MSYS%,$(shell uname))) -HOST_OS = MSYS -CMAKE_PARAMS = -G"MSYS Makefiles" -endif - - -#------------------------------------------------------------------------ -#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets -#------------------------------------------------------------------------ -ifneq (,$(filter $(HOST_OS),MSYS POSIX)) - -cmake: - @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE) - gpptest: clean g++ -v CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" diff --git a/contrib/gen_manual/.gitignore b/contrib/gen_manual/.gitignore new file mode 100644 index 0000000..6ea967f --- /dev/null +++ b/contrib/gen_manual/.gitignore @@ -0,0 +1,2 @@ +# build artefact +gen_manual diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile index adbcca2..9fbe858 100644 --- a/contrib/gen_manual/Makefile +++ b/contrib/gen_manual/Makefile @@ -35,7 +35,15 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -W CFLAGS += $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) - +LZ4API = ../../lib/lz4.h +LZ4MANUAL = ../../doc/lz4_manual.html +LZ4FAPI = ../../lib/lz4frame.h +LZ4FMANUAL = ../../doc/lz4frame_manual.html +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LZ4VER := $(shell echo $(LIBVER_SCRIPT)) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -45,14 +53,24 @@ EXT = endif -.PHONY: default gen_manual - +.PHONY: default default: gen_manual gen_manual: gen_manual.cpp - $(CXX) $(FLAGS) $^ -o $@$(EXT) + $(CXX) $(FLAGS) $^ -o $@$(EXT) +$(LZ4MANUAL) : gen_manual $(LZ4API) + echo "Update lz4 manual in /doc" + ./gen_manual $(LZ4VER) $(LZ4API) $@ +$(LZ4FMANUAL) : gen_manual $(LZ4FAPI) + echo "Update lz4frame manual in /doc" + ./gen_manual $(LZ4VER) $(LZ4FAPI) $@ + +.PHONY: manuals +manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL) + +.PHONY: clean clean: @$(RM) gen_manual$(EXT) @echo Cleaning completed diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index ff6e149..4c4174d 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -1,10 +1,10 @@ -lz4 1.7.5 Manual +1.7.6 Manual -

lz4 1.7.5 Manual

+

1.7.6 Manual


Contents

    @@ -170,21 +170,21 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr);


    -
    int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
    +
    int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
     

    Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - Important : Previous data blocks are assumed to still be present and unmodified ! + Important : Previous data blocks are assumed to remain present and unmodified ! 'dst' buffer must be already allocated. - If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + After an error, the stream status is invalid, and it can only be reset or freed.


    int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
    -

    If previously compressed data block is not guaranteed to remain available at its memory location, +

    If previously compressed data block is not guaranteed to remain available at its current memory location, save it into a safer place (char* safeBuffer). - Note : you don't need to call LZ4_loadDict() afterwards, - dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. + @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.


    diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index f76ec3d..5d553ed 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -1,19 +1,23 @@ -lz4frame 1.7.5 Manual +1.7.6 Manual -

    lz4frame 1.7.5 Manual

    +

    1.7.6 Manual


    Contents

    1. Introduction
    2. -
    3. Error management
    4. -
    5. Frame compression types
    6. -
    7. Simple compression function
    8. -
    9. Advanced compression functions
    10. -
    11. Decompression functions
    12. +
    13. Compiler specifics
    14. +
    15. Error management
    16. +
    17. Frame compression types
    18. +
    19. Simple compression function
    20. +
    21. Advanced compression functions
    22. +
    23. Resource Management
    24. +
    25. Compression
    26. +
    27. Decompression functions
    28. +
    29. Streaming decompression functions

    Introduction

    @@ -22,13 +26,15 @@
       of encoding standard metadata alongside LZ4-compressed blocks.
     
    -

    Error management

    
    +

    Compiler specifics

    
    +
    +

    Error management

    
     
     
    unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
     

    const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
     

    -

    Frame compression types

    
    +

    Frame compression types

    
     
     
    typedef enum {
         LZ4F_default=0,
    @@ -77,7 +83,7 @@
     
     
    typedef struct {
       LZ4F_frameInfo_t frameInfo;
    -  int      compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
    +  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
       unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
       unsigned reserved[4];            /* must be zero for forward compatibility */
     } LZ4F_preferences_t;
    @@ -86,7 +92,7 @@
      All reserved fields must be set to zero. 
     


    -

    Simple compression function

    
    +

    Simple compression function

    
     
     
    size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
     

    Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. @@ -105,17 +111,19 @@


    -

    Advanced compression functions

    
    +

    Advanced compression functions

    
     
     
    typedef struct {
       unsigned stableSrc;    /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */
       unsigned reserved[3];
     } LZ4F_compressOptions_t;
     

    +

    Resource Management

    
    +
     
    LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
     LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
     

    The first thing to do is to create a compressionContext object, which will be used in all compression operations. - This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. The function will provide a pointer to a fully allocated LZ4F_cctx object. If @return != zero, there was an error during context creation. @@ -123,6 +131,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);


    +

    Compression

    
    +
     
    size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
     

    will write the frame header into dstBuffer. dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. @@ -173,7 +183,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);


    -

    Decompression functions

    
    +

    Decompression functions

    
     
     
    typedef struct {
       unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
    @@ -181,7 +191,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
     } LZ4F_decompressOptions_t;
     

    LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
    -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
    +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
     

    Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations. The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions. The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. @@ -192,19 +202,27 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);


    +

    Streaming decompression functions

    
    +
     
    size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                          LZ4F_frameInfo_t* frameInfoPtr,
                                          const void* srcBuffer, size_t* srcSizePtr);
    -

    This function decodes frame header information (such as max blockSize, frame checksum, etc.). - Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. - A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that. +

    This function extracts frame parameters (such as max blockSize, frame checksum, etc.). + Its usage is optional. Extracted information can be useful for allocation purposes, typically. + This function works in 2 situations : + - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. + Input size must be large enough to successfully decode the entire frame header. + Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. + It's allowed to provide more input data than this minimum. + - After decoding has been started. + In which case, no input is read, frame parameters are extracted from dctx. + If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail. The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - Decompression must resume from this point (srcBuffer + *srcSizePtr). - Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough. - Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure. + Decompression must resume from (srcBuffer + *srcSizePtr). @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, or an error code which can be tested using LZ4F_isError() - (typically, when there is not enough src bytes to fully decode the frame header) + note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. + note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.


    @@ -227,14 +245,25 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx); @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. Schematically, it's the size of the current (or remaining) compressed block + header of next block. - Respecting the hint provides some boost to performance, since it does skip intermediate buffers. + Respecting the hint provides some small speed benefit, because it skips intermediate buffers. This is just a hint though, it's always possible to provide any srcSize. When a frame is fully decoded, @return will be 0 (no more data expected). If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). After a frame is fully decoded, dctx can be used again to decompress another frame. + After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.


    +
    LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
    +

    When decompression ends successfully, + it's possible to start a new decompression immediately + re-using the same context. + However, in case of an error, the context is left in "undefined" state. + In which case, it's necessary to reset it, before re-using it. + This method can also be used to abruptly stop an unfinished decompression, + and start a new on the same context. +


    + diff --git a/lib/Makefile b/lib/Makefile index 9a794b8..c6fd7b8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -46,10 +46,10 @@ BUILD_STATIC:= yes CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 -DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ +DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) @@ -70,12 +70,13 @@ endif LIBLZ4 = liblz4.$(SHARED_EXT_VER) +.PHONY: default default: lib-release -lib-release: liblz4.a liblz4 +lib-release: DEBUGFLAGS := +lib-release: lib -lib: CFLAGS += $(DEBUGFLAGS) -lib: lib-release +lib: liblz4.a liblz4 all: lib @@ -83,7 +84,7 @@ all32: CFLAGS+=-m32 all32: all liblz4.a: *.c -ifeq ($(BUILD_STATIC),yes) +ifeq ($(BUILD_STATIC),yes) # can be disabled on command line @echo compiling static library @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ @$(AR) rcs $@ *.o diff --git a/lib/lz4frame.h b/lib/lz4frame.h index b61fe7f..9a2130a 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -363,7 +363,7 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, const LZ4F_decompressOptions_t* dOptPtr); -/*! LZ4F_resetDecompressionContext() : +/*! LZ4F_resetDecompressionContext() : v1.8.0 * When decompression ends successfully, * it's possible to start a new decompression immediately * re-using the same context. diff --git a/programs/Makefile b/programs/Makefile index 4a8103c..2e73780 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -51,7 +51,7 @@ CFLAGS ?= -O3 DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) LZ4_VERSION=$(LIBVER) @@ -75,28 +75,17 @@ all: lz4 lz4c all32: CFLAGS+=-m32 all32: all -lz4: CFLAGS += $(DEBUGFLAGS) lz4: $(OBJFILES) $(CC) $(FLAGS) $^ -o $@$(EXT) lz4-release: DEBUGFLAGS= lz4-release: lz4 -lz4c : CFLAGS += $(DEBUGFLAGS) -lz4c : $(SRCFILES) - $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) - -lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) -lz4c32: $(SRCFILES) +lz4c32: CFLAGS += -m32 +lz4c: CPPFLAGS += -DENABLE_LZ4C_LEGACY_OPTIONS +lz4c lz4c32 : $(SRCFILES) $(CC) $(FLAGS) $^ -o $@$(EXT) -clean: - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) - @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat - @echo Cleaning completed - - lz4.1: lz4.1.md cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ @@ -108,6 +97,12 @@ clean-man: preview-man: clean-man man man ./lz4.1 +clean: + @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) + @$(RM) core *.o *.test tmp* \ + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat + @echo Cleaning completed + #----------------------------------------------------------------------------- # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets @@ -120,7 +115,6 @@ unlz4: lz4 lz4cat: lz4 ln -s lz4 lz4cat - ifneq (,$(filter $(shell uname),SunOS)) INSTALL ?= ginstall else From a902127613921408bb02b1debe9371e9bcfaf9f2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 May 2017 13:27:18 -0700 Subject: [PATCH 4/7] bumped version number to 1.8.0 due to addition of prototype LZ4F_resetDecompressionContext() --- doc/lz4_manual.html | 4 ++-- doc/lz4frame_manual.html | 4 ++-- lib/lz4.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 4c4174d..fecd8cd 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -1,10 +1,10 @@ -1.7.6 Manual +1.8.0 Manual -

    1.7.6 Manual

    +

    1.8.0 Manual


    Contents

      diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 5d553ed..fa2c0b9 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -1,10 +1,10 @@ -1.7.6 Manual +1.8.0 Manual -

      1.7.6 Manual

      +

      1.8.0 Manual


      Contents

        diff --git a/lib/lz4.h b/lib/lz4.h index d5a3016..5b6bc92 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -88,8 +88,8 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) From 2012e4de9e2c33a5abb34d40082afc8918852c6e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 May 2017 14:51:09 -0700 Subject: [PATCH 5/7] fixed c_standards tests and added entry "make list" --- Makefile | 24 ++++++++++++++---------- tests/frametest.c | 3 ++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 24328d5..b31ebd0 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,10 @@ endif default: lib lz4-release .PHONY: all -all: default examples manuals +all: allmost manuals + +.PHONY: allmost +allmost: lib lz4 examples .PHONY: lib lib: @@ -68,7 +71,7 @@ lz4 lz4-release: lib @cp $(PRGDIR)/lz4$(EXT) . .PHONY: examples -examples: lib lz4-release +examples: lib lz4 $(MAKE) -C $(EXDIR) test .PHONY: manuals @@ -116,6 +119,10 @@ endif #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + .PHONY: test test: $(MAKE) -C $(TESTDIR) $@ @@ -166,13 +173,10 @@ gpptest32: clean CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" c_standards: clean - $(MAKE) all MOREFLAGS="-std=gnu90 -Werror" - $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=c99 -Werror" - $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=gnu99 -Werror" - $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=c11 -Werror" - $(MAKE) clean + # note : lz4 is not C90 compatible, because it requires long long support + CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost + CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost + CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost + CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost endif diff --git a/tests/frametest.c b/tests/frametest.c index a4c548e..04597a4 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -290,7 +290,8 @@ int basicTests(U32 seed, double compressibility) { size_t nullSize = 0; size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize); if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { - DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError)); + DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", + LZ4F_getErrorName(fiError)); goto _output_error; } DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); From e60cbb5cacaa43eda7a3df7c2d9051a9437e7046 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 May 2017 16:28:36 -0700 Subject: [PATCH 6/7] added test for LZ4F_resetDecompressionContext() --- doc/lz4frame_manual.html | 9 ++---- lib/lz4frame.c | 66 ++++++++++++++++++++++++++-------------- lib/lz4frame.h | 9 ++---- lib/lz4frame_static.h | 9 ------ tests/frametest.c | 45 +++++++++++++++++---------- 5 files changed, 79 insertions(+), 59 deletions(-) diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index fa2c0b9..87750a1 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -255,14 +255,11 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);


    -
    LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
    -

    When decompression ends successfully, - it's possible to start a new decompression immediately - re-using the same context. - However, in case of an error, the context is left in "undefined" state. +

    void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
    +

    In case of an error, the context is left in "undefined" state. In which case, it's necessary to reset it, before re-using it. This method can also be used to abruptly stop an unfinished decompression, - and start a new on the same context. + and start a new with the same context.


    diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 7422638..fb37789 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -798,10 +798,9 @@ typedef enum { dstage_skipSkippable } dStage_t; -LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) +void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) { dctx->dStage = dstage_getHeader; - return 0; } @@ -939,7 +938,8 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn size_t o=0, i=0; *srcSizePtr = 0; *frameInfoPtr = dctxPtr->frameInfo; - return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */ + /* returns : recommended nb of bytes for LZ4F_decompress() */ + return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); } else { if (dctxPtr->dStage == dstage_storeHeader) { /* frame decoding already started, in the middle of header => automatic fail */ @@ -949,7 +949,10 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn size_t decodeResult; size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } - if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); } + if (*srcSizePtr < hSize) { + *srcSizePtr=0; + return err0r(LZ4F_ERROR_frameHeader_incomplete); + } decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize); if (LZ4F_isError(decodeResult)) { @@ -965,10 +968,15 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn /* trivial redirector, for common prototype */ -static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +static int LZ4F_decompress_safe (const char* src, + char* dst, + int compressedSize, + int dstCapacity, + const char* dictStart, + int dictSize) { (void)dictStart; (void)dictSize; - return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); + return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity); } @@ -1101,7 +1109,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, doAnotherStage = 0; /* not enough src data, ask for some more */ break; } - { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); + { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); /* will change dStage appropriately */ if (LZ4F_isError(hSize)) return hSize; } break; @@ -1149,7 +1157,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */ + if (dctxPtr->tmpInSize < BHSize) { /* not enough input for cBlockSize */ nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; doAnotherStage = 0; break; @@ -1157,13 +1165,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, selectedIn = dctxPtr->tmpIn; } - /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ + /* case dstage_decodeCBlockSize: */ /* no more direct access, to remove scan-build warning */ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */ dctxPtr->dStage = dstage_getSuffix; break; } - if (nextCBlockSize > dctxPtr->maxBlockSize) return err0r(LZ4F_ERROR_GENERIC); /* invalid cBlockSize */ + if (nextCBlockSize > dctxPtr->maxBlockSize) + return err0r(LZ4F_ERROR_maxBlockSize_invalid); dctxPtr->tmpInTarget = nextCBlockSize; if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { dctxPtr->dStage = dstage_copyDirect; @@ -1179,11 +1188,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, case dstage_copyDirect: /* uncompressed block */ { size_t sizeToCopy = dctxPtr->tmpInTarget; - if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ + if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; + if (dctxPtr->frameInfo.contentChecksumFlag) + XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentSize) + dctxPtr->frameRemainingSize -= sizeToCopy; /* dictionary management */ if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) @@ -1244,10 +1255,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, else decoder = LZ4F_decompress_safe; - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, + (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, + (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + if (dctxPtr->frameInfo.contentChecksumFlag) + XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + if (dctxPtr->frameInfo.contentSize) + dctxPtr->frameRemainingSize -= decodedSize; /* dictionary management */ if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) @@ -1284,10 +1299,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, } /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, + (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, + (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) + return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) + XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + if (dctxPtr->frameInfo.contentSize) + dctxPtr->frameRemainingSize -= decodedSize; dctxPtr->tmpOutSize = decodedSize; dctxPtr->tmpOutStart = 0; dctxPtr->dStage = dstage_flushOut; @@ -1318,7 +1338,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, case dstage_getSuffix: { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (dctxPtr->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ + if (dctxPtr->frameRemainingSize) + return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ if (suffixSize == 0) { /* frame completed */ nextSrcSizeHint = 0; dctxPtr->dStage = dstage_getHeader; @@ -1379,7 +1400,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */ + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { + /* not enough input to get full sBlockSize; wait for more */ nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; doAnotherStage = 0; break; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 9a2130a..b1719e2 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -364,14 +364,11 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, /*! LZ4F_resetDecompressionContext() : v1.8.0 - * When decompression ends successfully, - * it's possible to start a new decompression immediately - * re-using the same context. - * However, in case of an error, the context is left in "undefined" state. + * In case of an error, the context is left in "undefined" state. * In which case, it's necessary to reset it, before re-using it. * This method can also be used to abruptly stop an unfinished decompression, - * and start a new on the same context. */ -LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); + * and start a new with the same context. */ +LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index 8ea496d..d3bae82 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -50,15 +50,6 @@ extern "C" { #include "lz4frame.h" -/* --- Experimental functions --- */ -/* LZ4F_resetDecompressionContext() : - * LZ4F_decompress() does not guarantee to leave dctx in clean state in case of errors. - * In order to re-use a dctx after a decompression error, - * use LZ4F_resetDecompressionContext() first. - * dctx will be able to start decompression on a new frame */ -LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); - - /* --- Error List --- */ #define LZ4F_LIST_ERRORS(ITEM) \ ITEM(OK_NoError) \ diff --git a/tests/frametest.c b/tests/frametest.c index 04597a4..538bedc 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -40,6 +40,7 @@ #include /* fprintf */ #include /* strcmp */ #include /* clock_t, clock(), CLOCKS_PER_SEC */ +#include #include "lz4frame_static.h" #include "lz4.h" /* LZ4_VERSION_STRING */ #define XXH_STATIC_LINKING_ONLY @@ -67,7 +68,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) #define GB *(1U<<30) static const U32 nbTestsDefault = 256 KB; -#define COMPRESSIBLE_NOISE_LENGTH (2 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -166,7 +166,7 @@ static unsigned FUZ_highbit(U32 v32) *********************************************************/ int basicTests(U32 seed, double compressibility) { - int testResult = 0; +#define COMPRESSIBLE_NOISE_LENGTH (2 MB) void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); void* const compressedBuffer = malloc(cBuffSize); @@ -176,9 +176,10 @@ int basicTests(U32 seed, double compressibility) LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cctx = NULL; U64 crcOrig; - + int basicTests_error = 0; LZ4F_preferences_t prefs; memset(&prefs, 0, sizeof(prefs)); + if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); goto _output_error; @@ -198,7 +199,7 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : "); cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); + DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize); DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); @@ -236,8 +237,6 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Decompression test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; size_t compressedBufferSize = cSize; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) goto _output_error; @@ -280,6 +279,7 @@ int basicTests(U32 seed, double compressibility) { size_t oSize = 0; size_t iSize = 0; LZ4F_frameInfo_t fi; + const BYTE* ip = (BYTE*)compressedBuffer; DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); @@ -315,10 +315,30 @@ int basicTests(U32 seed, double compressibility) ip += iSize; } + DISPLAYLEVEL(3, "Decode a buggy input : "); + assert(COMPRESSIBLE_NOISE_LENGTH > 64); + assert(cSize > 48); + memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */ + memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */ + { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH; + size_t cbSize = cSize; + size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize, + compressedBuffer, &cbSize, + NULL); + if (!LZ4F_isError(decompressError)) goto _output_error; + DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError)); + } + memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */ + + DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n"); + LZ4F_resetDecompressionContext(dCtx); /* always successful */ + DISPLAYLEVEL(3, "Byte after byte : "); { BYTE* const ostart = (BYTE*)decodedBuffer; BYTE* op = ostart; BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + const BYTE* ip = compressedBuffer; + const BYTE* const iend = ip + cSize; while (ip < iend) { size_t oSize = oend-op; size_t iSize = 1; @@ -330,11 +350,7 @@ int basicTests(U32 seed, double compressibility) { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (crcDest != crcOrig) goto _output_error; } DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); - } - - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; - dCtx = NULL; + } } DISPLAYLEVEL(3, "Using 64 KB block : "); @@ -366,9 +382,6 @@ int basicTests(U32 seed, double compressibility) const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; - { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(createError)) goto _output_error; } - DISPLAYLEVEL(3, "random segment sizes : "); while (ip < iend) { unsigned const nbBits = FUZ_rand(&randState) % maxBits; @@ -552,10 +565,10 @@ _end: free(decodedBuffer); LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; LZ4F_freeCompressionContext(cctx); cctx = NULL; - return testResult; + return basicTests_error; _output_error: - testResult = 1; + basicTests_error = 1; DISPLAY("Error detected ! \n"); goto _end; } From 2600a154bebdf79637e289ed19e53e2c72a168d7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 May 2017 17:06:31 -0700 Subject: [PATCH 7/7] fix (minor) g++ compatibility for frametest --- tests/frametest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frametest.c b/tests/frametest.c index 538bedc..0dadf9f 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -337,7 +337,7 @@ int basicTests(U32 seed, double compressibility) { BYTE* const ostart = (BYTE*)decodedBuffer; BYTE* op = ostart; BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - const BYTE* ip = compressedBuffer; + const BYTE* ip = (const BYTE*) compressedBuffer; const BYTE* const iend = ip + cSize; while (ip < iend) { size_t oSize = oend-op;